diff --git "a/train.log" "b/train.log" new file mode 100644--- /dev/null +++ "b/train.log" @@ -0,0 +1,2313 @@ +# python3 -m espnet2.bin.asr_unimodal_train --use_preprocessor true --bpemodel none --token_type char --token_list data/zh_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev/wav.scp,speech,kaldi_ark --valid_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/speech_shape --resume true --ignore_init_mismatch false --fold_length 51200 --output_dir exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp --config conf/train_asr_unimodal_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/train_sp/wav.scp,speech,kaldi_ark --train_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/train_sp/text,text,text --train_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/dev/text,text,text --valid_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/text_shape.char --ngpu 1 --multiprocessing_distributed True +# Started at Fri Jun 9 12:37:13 CST 2023 +# +/data/home/fangying/anaconda3/envs/espnet/bin/python3 /data/home/fangying/espnet/espnet2/bin/asr_unimodal_train.py --use_preprocessor true --bpemodel none --token_type char --token_list data/zh_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev/wav.scp,speech,kaldi_ark --valid_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/speech_shape --resume true --ignore_init_mismatch false --fold_length 51200 --output_dir exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp --config conf/train_asr_unimodal_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/train_sp/wav.scp,speech,kaldi_ark --train_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/train_sp/text,text,text --train_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/dev/text,text,text --valid_shape_file exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/text_shape.char --ngpu 1 --multiprocessing_distributed True +[ALab02] 2023-06-09 12:37:15,857 (asr_unimodal:512) INFO: Vocabulary size: 4233 +[ALab02] 2023-06-09 12:37:18,897 (abs_task:1201) INFO: pytorch.version=1.12.1, cuda.available=True, cudnn.version=8302, cudnn.benchmark=False, cudnn.deterministic=True +[ALab02] 2023-06-09 12:37:18,903 (abs_task:1202) INFO: Model structure: +UAMASRModel( + (frontend): DefaultFrontend( + (stft): Stft(n_fft=512, win_length=512, hop_length=128, center=True, normalized=False, onesided=True) + (frontend): Frontend() + (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False) + ) + (specaug): SpecAug( + (time_warp): TimeWarp(window=5, mode=bicubic) + (freq_mask): MaskAlongAxis(mask_width_range=[0, 27], num_mask=2, axis=freq) + (time_mask): MaskAlongAxisVariableMaxWidth(mask_width_ratio_range=[0.0, 0.05], num_mask=10, axis=time) + ) + (normalize): GlobalMVN(stats_file=exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/feats_stats.npz, norm_means=True, norm_vars=True) + (encoder): ConformerEncoder( + (embed): Conv2dSubsampling( + (conv): Sequential( + (0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2)) + (1): ReLU() + (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2)) + (3): ReLU() + ) + (out): Sequential( + (0): Linear(in_features=4864, out_features=256, bias=True) + (1): RelPositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (6): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (7): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (8): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (9): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (10): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (11): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + ) + (uma): UMA( + (linear_sigmoid): Sequential( + (0): Linear(in_features=256, out_features=1, bias=True) + (1): Sigmoid() + ) + ) + (decoder): UnimodalAttentionDecoder( + (embed): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (2): Dropout(p=0.1, inplace=False) + (3): ReLU() + (4): PositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.0, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + ) + (criterion_att): LabelSmoothingLoss( + (criterion): KLDivLoss() + ) + (ctc): CTC( + (ctc_lo): Linear(in_features=256, out_features=4233, bias=True) + (ctc_loss): CTCLoss() + ) +) + +Model summary: + Class Name: UAMASRModel + Total Number of model parameters: 42.56 M + Number of trainable parameters: 42.56 M (100.0%) + Size: 170.24 MB + Type: torch.float32 +[ALab02] 2023-06-09 12:37:18,903 (abs_task:1205) INFO: Optimizer: +Adam ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.999) + capturable: False + eps: 1e-08 + foreach: None + initial_lr: 0.0005 + lr: 1.4285714285714288e-08 + maximize: False + weight_decay: 1e-06 +) +[ALab02] 2023-06-09 12:37:18,903 (abs_task:1206) INFO: Scheduler: WarmupLR(warmup_steps=35000) +[ALab02] 2023-06-09 12:37:18,903 (abs_task:1215) INFO: Saving the configuration in exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/config.yaml +[ALab02] 2023-06-09 12:37:19,775 (asr_unimodal:483) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[ALab02] 2023-06-09 12:37:21,798 (abs_task:1570) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/train_sp/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/train_sp/text", "type": "text"} + preprocess: ) +[ALab02] 2023-06-09 12:37:21,798 (abs_task:1571) INFO: [train] Batch sampler: FoldedBatchSampler(N-batch=11029, batch_size=64, shape_files=['exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/speech_shape', 'exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/train/text_shape.char'], sort_in_batch=descending, sort_batch=descending) +[ALab02] 2023-06-09 12:37:21,799 (abs_task:1572) INFO: [train] mini-batch sizes summary: N-batch=11029, mean=32.7, min=4, max=64 +[ALab02] 2023-06-09 12:37:21,851 (asr_unimodal:483) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[ALab02] 2023-06-09 12:37:21,975 (abs_task:1570) INFO: [valid] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/dev/text", "type": "text"} + preprocess: ) +[ALab02] 2023-06-09 12:37:21,975 (abs_task:1571) INFO: [valid] Batch sampler: FoldedBatchSampler(N-batch=441, batch_size=64, shape_files=['exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/speech_shape', 'exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/text_shape.char'], sort_in_batch=descending, sort_batch=descending) +[ALab02] 2023-06-09 12:37:21,975 (abs_task:1572) INFO: [valid] mini-batch sizes summary: N-batch=441, mean=32.5, min=6, max=64 +[ALab02] 2023-06-09 12:37:21,992 (asr_unimodal:483) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[ALab02] 2023-06-09 12:37:22,020 (abs_task:1570) INFO: [plot_att] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/dev/text", "type": "text"} + preprocess: ) +[ALab02] 2023-06-09 12:37:22,021 (abs_task:1571) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=14326, batch_size=1, key_file=exp_uma_conformer_12e_69/asr_stats_raw_zh_char_sp/valid/speech_shape, +[ALab02] 2023-06-09 12:37:22,021 (abs_task:1572) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1 +[ALab02] 2023-06-09 12:37:22,228 (trainer:284) INFO: 1/70epoch started +[ALab02] 2023-06-09 12:39:57,248 (trainer:721) INFO: 1epoch:train:1-551batch: iter_time=9.204e-04, forward_time=0.134, loss_ctc=76.255, loss=76.255, backward_time=0.094, optim_step_time=0.034, optim0_lr0=1.000e-06, train_time=1.126 +[ALab02] 2023-06-09 12:42:28,720 (trainer:721) INFO: 1epoch:train:552-1102batch: iter_time=1.747e-04, forward_time=0.129, loss_ctc=40.835, loss=40.835, backward_time=0.094, optim_step_time=0.033, optim0_lr0=2.964e-06, train_time=1.100 +[ALab02] 2023-06-09 12:44:39,348 (trainer:721) INFO: 1epoch:train:1103-1653batch: iter_time=1.764e-04, forward_time=0.107, loss_ctc=33.498, loss=33.498, backward_time=0.086, optim_step_time=0.034, optim0_lr0=4.936e-06, train_time=0.948 +[ALab02] 2023-06-09 12:47:02,785 (trainer:721) INFO: 1epoch:train:1654-2204batch: iter_time=1.699e-04, forward_time=0.120, loss_ctc=28.807, loss=28.807, backward_time=0.092, optim_step_time=0.034, optim0_lr0=6.907e-06, train_time=1.040 +[ALab02] 2023-06-09 12:49:36,575 (trainer:721) INFO: 1epoch:train:2205-2755batch: iter_time=1.759e-04, forward_time=0.130, loss_ctc=26.609, loss=26.609, backward_time=0.096, optim_step_time=0.034, optim0_lr0=8.871e-06, train_time=1.117 +[ALab02] 2023-06-09 12:52:34,874 (trainer:721) INFO: 1epoch:train:2756-3306batch: iter_time=3.600e-04, forward_time=0.152, loss_ctc=25.983, loss=25.983, backward_time=0.117, optim_step_time=0.047, optim0_lr0=1.084e-05, train_time=1.294 +[ALab02] 2023-06-09 12:55:32,293 (trainer:721) INFO: 1epoch:train:3307-3857batch: iter_time=2.660e-04, forward_time=0.150, loss_ctc=25.416, loss=25.416, backward_time=0.118, optim_step_time=0.046, optim0_lr0=1.281e-05, train_time=1.286 +[ALab02] 2023-06-09 12:58:04,602 (trainer:721) INFO: 1epoch:train:3858-4408batch: iter_time=1.787e-04, forward_time=0.130, loss_ctc=24.531, loss=24.531, backward_time=0.094, optim_step_time=0.033, optim0_lr0=1.478e-05, train_time=1.106 +[ALab02] 2023-06-09 13:00:38,405 (trainer:721) INFO: 1epoch:train:4409-4959batch: iter_time=1.944e-04, forward_time=0.132, loss_ctc=24.955, loss=24.955, backward_time=0.096, optim_step_time=0.035, optim0_lr0=1.674e-05, train_time=1.118 +[ALab02] 2023-06-09 13:03:13,752 (trainer:721) INFO: 1epoch:train:4960-5510batch: iter_time=1.911e-04, forward_time=0.134, loss_ctc=24.751, loss=24.751, backward_time=0.096, optim_step_time=0.034, optim0_lr0=1.871e-05, train_time=1.126 +[ALab02] 2023-06-09 13:05:47,547 (trainer:721) INFO: 1epoch:train:5511-6061batch: iter_time=1.946e-04, forward_time=0.131, loss_ctc=24.700, loss=24.700, backward_time=0.095, optim_step_time=0.035, optim0_lr0=2.068e-05, train_time=1.116 +[ALab02] 2023-06-09 13:08:21,056 (trainer:721) INFO: 1epoch:train:6062-6612batch: iter_time=1.845e-04, forward_time=0.130, loss_ctc=24.369, loss=24.369, backward_time=0.096, optim_step_time=0.035, optim0_lr0=2.265e-05, train_time=1.114 +[ALab02] 2023-06-09 13:10:55,607 (trainer:721) INFO: 1epoch:train:6613-7163batch: iter_time=1.909e-04, forward_time=0.133, loss_ctc=24.564, loss=24.564, backward_time=0.096, optim_step_time=0.035, optim0_lr0=2.461e-05, train_time=1.121 +[ALab02] 2023-06-09 13:13:27,812 (trainer:721) INFO: 1epoch:train:7164-7714batch: iter_time=1.913e-04, forward_time=0.130, loss_ctc=24.589, loss=24.589, backward_time=0.094, optim_step_time=0.035, optim0_lr0=2.658e-05, train_time=1.106 +[ALab02] 2023-06-09 13:15:23,519 (trainer:721) INFO: 1epoch:train:7715-8265batch: iter_time=1.913e-04, forward_time=0.089, loss_ctc=23.959, loss=23.959, backward_time=0.083, optim_step_time=0.035, optim0_lr0=2.855e-05, train_time=0.841 +[ALab02] 2023-06-09 13:17:04,530 (trainer:721) INFO: 1epoch:train:8266-8816batch: iter_time=2.112e-04, forward_time=0.068, loss_ctc=23.582, loss=23.582, backward_time=0.086, optim_step_time=0.042, optim0_lr0=3.052e-05, train_time=0.733 +[ALab02] 2023-06-09 13:18:23,980 (trainer:721) INFO: 1epoch:train:8817-9367batch: iter_time=1.512e-04, forward_time=0.051, loss_ctc=24.228, loss=24.228, backward_time=0.067, optim_step_time=0.035, optim0_lr0=3.249e-05, train_time=0.577 +[ALab02] 2023-06-09 13:19:56,056 (trainer:721) INFO: 1epoch:train:9368-9918batch: iter_time=1.891e-04, forward_time=0.061, loss_ctc=23.180, loss=23.180, backward_time=0.077, optim_step_time=0.037, optim0_lr0=3.445e-05, train_time=0.668 +[ALab02] 2023-06-09 13:21:24,608 (trainer:721) INFO: 1epoch:train:9919-10469batch: iter_time=1.900e-04, forward_time=0.058, loss_ctc=22.321, loss=22.321, backward_time=0.074, optim_step_time=0.035, optim0_lr0=3.642e-05, train_time=0.643 +[ALab02] 2023-06-09 13:22:47,968 (trainer:721) INFO: 1epoch:train:10470-11020batch: iter_time=1.915e-04, forward_time=0.055, loss_ctc=22.053, loss=22.053, backward_time=0.071, optim_step_time=0.035, optim0_lr0=3.839e-05, train_time=0.605 +[ALab02] 2023-06-09 13:24:02,172 (trainer:338) INFO: 1epoch results: [train] iter_time=2.345e-04, forward_time=0.111, loss_ctc=28.455, loss=28.455, backward_time=0.091, optim_step_time=0.036, optim0_lr0=1.971e-05, train_time=0.989, time=45 minutes and 27.29 seconds, total_count=11029, gpu_max_cached_mem_GB=7.561, [valid] loss_ctc=83.201, cer_ctc=0.912, cer=0.912, loss=83.201, time=37.55 seconds, total_count=441, gpu_max_cached_mem_GB=10.844, [att_plot] time=35.07 seconds, total_count=0, gpu_max_cached_mem_GB=10.844 +[ALab02] 2023-06-09 13:24:04,368 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 13:24:04,369 (trainer:272) INFO: 2/70epoch started. Estimated time to finish: 2 days, 5 hours and 42 minutes +[ALab02] 2023-06-09 13:25:33,202 (trainer:721) INFO: 2epoch:train:1-551batch: iter_time=9.387e-04, forward_time=0.057, loss_ctc=21.864, loss=21.864, backward_time=0.076, optim_step_time=0.035, optim0_lr0=4.039e-05, train_time=0.644 +[ALab02] 2023-06-09 13:27:14,012 (trainer:721) INFO: 2epoch:train:552-1102batch: iter_time=2.027e-04, forward_time=0.070, loss_ctc=21.767, loss=21.767, backward_time=0.080, optim_step_time=0.035, optim0_lr0=4.235e-05, train_time=0.732 +[ALab02] 2023-06-09 13:28:55,262 (trainer:721) INFO: 2epoch:train:1103-1653batch: iter_time=2.091e-04, forward_time=0.070, loss_ctc=20.545, loss=20.545, backward_time=0.082, optim_step_time=0.035, optim0_lr0=4.432e-05, train_time=0.734 +[ALab02] 2023-06-09 13:30:32,920 (trainer:721) INFO: 2epoch:train:1654-2204batch: iter_time=1.848e-04, forward_time=0.066, loss_ctc=20.210, loss=20.210, backward_time=0.081, optim_step_time=0.035, optim0_lr0=4.629e-05, train_time=0.708 +[ALab02] 2023-06-09 13:32:08,466 (trainer:721) INFO: 2epoch:train:2205-2755batch: iter_time=1.920e-04, forward_time=0.063, loss_ctc=19.634, loss=19.634, backward_time=0.081, optim_step_time=0.036, optim0_lr0=4.826e-05, train_time=0.694 +[ALab02] 2023-06-09 13:33:49,779 (trainer:721) INFO: 2epoch:train:2756-3306batch: iter_time=1.933e-04, forward_time=0.070, loss_ctc=19.151, loss=19.151, backward_time=0.082, optim_step_time=0.035, optim0_lr0=5.022e-05, train_time=0.735 +[ALab02] 2023-06-09 13:35:26,836 (trainer:721) INFO: 2epoch:train:3307-3857batch: iter_time=1.956e-04, forward_time=0.066, loss_ctc=19.381, loss=19.381, backward_time=0.080, optim_step_time=0.035, optim0_lr0=5.219e-05, train_time=0.704 +[ALab02] 2023-06-09 13:37:06,829 (trainer:721) INFO: 2epoch:train:3858-4408batch: iter_time=1.989e-04, forward_time=0.068, loss_ctc=18.727, loss=18.727, backward_time=0.082, optim_step_time=0.035, optim0_lr0=5.416e-05, train_time=0.725 +[ALab02] 2023-06-09 13:38:51,515 (trainer:721) INFO: 2epoch:train:4409-4959batch: iter_time=2.053e-04, forward_time=0.074, loss_ctc=17.700, loss=17.700, backward_time=0.083, optim_step_time=0.036, optim0_lr0=5.613e-05, train_time=0.761 +[ALab02] 2023-06-09 13:40:33,203 (trainer:721) INFO: 2epoch:train:4960-5510batch: iter_time=1.808e-04, forward_time=0.072, loss_ctc=17.678, loss=17.678, backward_time=0.080, optim_step_time=0.035, optim0_lr0=5.809e-05, train_time=0.738 +[ALab02] 2023-06-09 13:42:07,121 (trainer:721) INFO: 2epoch:train:5511-6061batch: iter_time=1.802e-04, forward_time=0.063, loss_ctc=17.293, loss=17.293, backward_time=0.077, optim_step_time=0.035, optim0_lr0=6.006e-05, train_time=0.682 +[ALab02] 2023-06-09 13:43:51,671 (trainer:721) INFO: 2epoch:train:6062-6612batch: iter_time=1.776e-04, forward_time=0.074, loss_ctc=16.401, loss=16.401, backward_time=0.082, optim_step_time=0.034, optim0_lr0=6.204e-05, train_time=0.758 +[ALab02] 2023-06-09 13:45:30,685 (trainer:721) INFO: 2epoch:train:6613-7163batch: iter_time=1.695e-04, forward_time=0.069, loss_ctc=16.061, loss=16.061, backward_time=0.079, optim_step_time=0.034, optim0_lr0=6.400e-05, train_time=0.719 +[ALab02] 2023-06-09 13:47:05,812 (trainer:721) INFO: 2epoch:train:7164-7714batch: iter_time=1.719e-04, forward_time=0.065, loss_ctc=16.024, loss=16.024, backward_time=0.077, optim_step_time=0.034, optim0_lr0=6.596e-05, train_time=0.690 +[ALab02] 2023-06-09 13:48:53,998 (trainer:721) INFO: 2epoch:train:7715-8265batch: iter_time=1.716e-04, forward_time=0.079, loss_ctc=15.320, loss=15.320, backward_time=0.081, optim_step_time=0.034, optim0_lr0=6.794e-05, train_time=0.785 +[ALab02] 2023-06-09 13:50:36,961 (trainer:721) INFO: 2epoch:train:8266-8816batch: iter_time=1.824e-04, forward_time=0.073, loss_ctc=14.887, loss=14.887, backward_time=0.081, optim_step_time=0.035, optim0_lr0=6.991e-05, train_time=0.748 +[ALab02] 2023-06-09 13:52:16,646 (trainer:721) INFO: 2epoch:train:8817-9367batch: iter_time=2.070e-04, forward_time=0.068, loss_ctc=14.460, loss=14.460, backward_time=0.081, optim_step_time=0.035, optim0_lr0=7.187e-05, train_time=0.724 +[ALab02] 2023-06-09 13:53:55,242 (trainer:721) INFO: 2epoch:train:9368-9918batch: iter_time=2.082e-04, forward_time=0.067, loss_ctc=14.571, loss=14.571, backward_time=0.082, optim_step_time=0.036, optim0_lr0=7.384e-05, train_time=0.715 +[ALab02] 2023-06-09 13:55:32,012 (trainer:721) INFO: 2epoch:train:9919-10469batch: iter_time=2.145e-04, forward_time=0.065, loss_ctc=13.863, loss=13.863, backward_time=0.080, optim_step_time=0.036, optim0_lr0=7.581e-05, train_time=0.702 +[ALab02] 2023-06-09 13:58:01,258 (trainer:721) INFO: 2epoch:train:10470-11020batch: iter_time=3.863e-04, forward_time=0.108, loss_ctc=13.655, loss=13.655, backward_time=0.123, optim_step_time=0.055, optim0_lr0=7.778e-05, train_time=1.082 +[ALab02] 2023-06-09 13:59:00,031 (trainer:338) INFO: 2epoch results: [train] iter_time=2.384e-04, forward_time=0.070, loss_ctc=17.457, loss=17.457, backward_time=0.082, optim_step_time=0.036, optim0_lr0=5.910e-05, train_time=0.739, time=33 minutes and 58.61 seconds, total_count=22058, gpu_max_cached_mem_GB=10.846, [valid] loss_ctc=38.601, cer_ctc=0.534, cer=0.534, loss=38.601, time=25.43 seconds, total_count=882, gpu_max_cached_mem_GB=10.846, [att_plot] time=31.62 seconds, total_count=0, gpu_max_cached_mem_GB=10.846 +[ALab02] 2023-06-09 13:59:02,447 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 13:59:02,447 (trainer:272) INFO: 3/70epoch started. Estimated time to finish: 1 day, 22 hours and 16 minutes +[ALab02] 2023-06-09 14:00:43,932 (trainer:721) INFO: 3epoch:train:1-551batch: iter_time=0.002, forward_time=0.070, loss_ctc=12.934, loss=12.934, backward_time=0.081, optim_step_time=0.036, optim0_lr0=7.977e-05, train_time=0.735 +[ALab02] 2023-06-09 14:02:30,681 (trainer:721) INFO: 3epoch:train:552-1102batch: iter_time=2.992e-04, forward_time=0.075, loss_ctc=13.195, loss=13.195, backward_time=0.084, optim_step_time=0.036, optim0_lr0=8.174e-05, train_time=0.777 +[ALab02] 2023-06-09 14:04:10,946 (trainer:721) INFO: 3epoch:train:1103-1653batch: iter_time=2.012e-04, forward_time=0.069, loss_ctc=12.511, loss=12.511, backward_time=0.081, optim_step_time=0.035, optim0_lr0=8.371e-05, train_time=0.728 +[ALab02] 2023-06-09 14:05:52,523 (trainer:721) INFO: 3epoch:train:1654-2204batch: iter_time=1.969e-04, forward_time=0.070, loss_ctc=12.154, loss=12.154, backward_time=0.082, optim_step_time=0.036, optim0_lr0=8.568e-05, train_time=0.737 +[ALab02] 2023-06-09 14:07:37,902 (trainer:721) INFO: 3epoch:train:2205-2755batch: iter_time=1.998e-04, forward_time=0.075, loss_ctc=11.540, loss=11.540, backward_time=0.083, optim_step_time=0.036, optim0_lr0=8.764e-05, train_time=0.764 +[ALab02] 2023-06-09 14:09:22,858 (trainer:721) INFO: 3epoch:train:2756-3306batch: iter_time=2.031e-04, forward_time=0.073, loss_ctc=11.464, loss=11.464, backward_time=0.085, optim_step_time=0.036, optim0_lr0=8.961e-05, train_time=0.763 +[ALab02] 2023-06-09 14:10:59,694 (trainer:721) INFO: 3epoch:train:3307-3857batch: iter_time=1.821e-04, forward_time=0.068, loss_ctc=11.480, loss=11.480, backward_time=0.078, optim_step_time=0.034, optim0_lr0=9.158e-05, train_time=0.702 +[ALab02] 2023-06-09 14:12:45,792 (trainer:721) INFO: 3epoch:train:3858-4408batch: iter_time=2.210e-04, forward_time=0.073, loss_ctc=11.220, loss=11.220, backward_time=0.086, optim_step_time=0.037, optim0_lr0=9.355e-05, train_time=0.770 +[ALab02] 2023-06-09 14:14:33,832 (trainer:721) INFO: 3epoch:train:4409-4959batch: iter_time=2.064e-04, forward_time=0.077, loss_ctc=10.706, loss=10.706, backward_time=0.085, optim_step_time=0.035, optim0_lr0=9.551e-05, train_time=0.785 +[ALab02] 2023-06-09 14:16:27,969 (trainer:721) INFO: 3epoch:train:4960-5510batch: iter_time=1.955e-04, forward_time=0.085, loss_ctc=10.587, loss=10.587, backward_time=0.085, optim_step_time=0.035, optim0_lr0=9.748e-05, train_time=0.828 +[ALab02] 2023-06-09 14:18:06,568 (trainer:721) INFO: 3epoch:train:5511-6061batch: iter_time=2.038e-04, forward_time=0.067, loss_ctc=10.293, loss=10.293, backward_time=0.081, optim_step_time=0.036, optim0_lr0=9.945e-05, train_time=0.715 +[ALab02] 2023-06-09 14:19:46,183 (trainer:721) INFO: 3epoch:train:6062-6612batch: iter_time=2.036e-04, forward_time=0.069, loss_ctc=9.966, loss=9.966, backward_time=0.081, optim_step_time=0.036, optim0_lr0=1.014e-04, train_time=0.723 +[ALab02] 2023-06-09 14:21:32,625 (trainer:721) INFO: 3epoch:train:6613-7163batch: iter_time=2.052e-04, forward_time=0.077, loss_ctc=9.928, loss=9.928, backward_time=0.083, optim_step_time=0.036, optim0_lr0=1.034e-04, train_time=0.774 +[ALab02] 2023-06-09 14:23:11,014 (trainer:721) INFO: 3epoch:train:7164-7714batch: iter_time=1.854e-04, forward_time=0.068, loss_ctc=9.723, loss=9.723, backward_time=0.080, optim_step_time=0.036, optim0_lr0=1.053e-04, train_time=0.713 +[ALab02] 2023-06-09 14:25:06,153 (trainer:721) INFO: 3epoch:train:7715-8265batch: iter_time=2.113e-04, forward_time=0.084, loss_ctc=9.914, loss=9.914, backward_time=0.086, optim_step_time=0.036, optim0_lr0=1.073e-04, train_time=0.836 +[ALab02] 2023-06-09 14:26:36,077 (trainer:721) INFO: 3epoch:train:8266-8816batch: iter_time=2.046e-04, forward_time=0.058, loss_ctc=9.423, loss=9.423, backward_time=0.079, optim_step_time=0.035, optim0_lr0=1.093e-04, train_time=0.653 +[ALab02] 2023-06-09 14:28:03,630 (trainer:721) INFO: 3epoch:train:8817-9367batch: iter_time=1.939e-04, forward_time=0.055, loss_ctc=9.213, loss=9.213, backward_time=0.078, optim_step_time=0.036, optim0_lr0=1.113e-04, train_time=0.636 +[ALab02] 2023-06-09 14:29:31,929 (trainer:721) INFO: 3epoch:train:9368-9918batch: iter_time=2.010e-04, forward_time=0.055, loss_ctc=9.144, loss=9.144, backward_time=0.079, optim_step_time=0.037, optim0_lr0=1.132e-04, train_time=0.640 +[ALab02] 2023-06-09 14:31:00,315 (trainer:721) INFO: 3epoch:train:9919-10469batch: iter_time=1.995e-04, forward_time=0.055, loss_ctc=8.679, loss=8.679, backward_time=0.079, optim_step_time=0.036, optim0_lr0=1.152e-04, train_time=0.642 +[ALab02] 2023-06-09 14:32:26,566 (trainer:721) INFO: 3epoch:train:10470-11020batch: iter_time=1.842e-04, forward_time=0.055, loss_ctc=8.757, loss=8.757, backward_time=0.076, optim_step_time=0.037, optim0_lr0=1.172e-04, train_time=0.626 +[ALab02] 2023-06-09 14:33:23,837 (trainer:338) INFO: 3epoch results: [train] iter_time=2.704e-04, forward_time=0.069, loss_ctc=10.640, loss=10.640, backward_time=0.082, optim_step_time=0.036, optim0_lr0=9.849e-05, train_time=0.727, time=33 minutes and 25.77 seconds, total_count=33087, gpu_max_cached_mem_GB=10.857, [valid] loss_ctc=18.550, cer_ctc=0.296, cer=0.296, loss=18.550, time=25.1 seconds, total_count=1323, gpu_max_cached_mem_GB=10.857, [att_plot] time=30.52 seconds, total_count=0, gpu_max_cached_mem_GB=10.857 +[ALab02] 2023-06-09 14:33:26,428 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 14:33:26,429 (trainer:272) INFO: 4/70epoch started. Estimated time to finish: 1 day, 19 hours and 12 minutes +[ALab02] 2023-06-09 14:34:57,558 (trainer:721) INFO: 4epoch:train:1-551batch: iter_time=7.923e-04, forward_time=0.058, loss_ctc=8.404, loss=8.404, backward_time=0.081, optim_step_time=0.037, optim0_lr0=1.192e-04, train_time=0.661 +[ALab02] 2023-06-09 14:36:26,365 (trainer:721) INFO: 4epoch:train:552-1102batch: iter_time=2.182e-04, forward_time=0.056, loss_ctc=8.419, loss=8.419, backward_time=0.078, optim_step_time=0.037, optim0_lr0=1.211e-04, train_time=0.645 +[ALab02] 2023-06-09 14:37:56,324 (trainer:721) INFO: 4epoch:train:1103-1653batch: iter_time=2.243e-04, forward_time=0.057, loss_ctc=8.384, loss=8.384, backward_time=0.079, optim_step_time=0.038, optim0_lr0=1.231e-04, train_time=0.653 +[ALab02] 2023-06-09 14:39:26,029 (trainer:721) INFO: 4epoch:train:1654-2204batch: iter_time=2.064e-04, forward_time=0.057, loss_ctc=8.059, loss=8.059, backward_time=0.079, optim_step_time=0.037, optim0_lr0=1.251e-04, train_time=0.651 +[ALab02] 2023-06-09 14:40:55,555 (trainer:721) INFO: 4epoch:train:2205-2755batch: iter_time=2.062e-04, forward_time=0.056, loss_ctc=8.080, loss=8.080, backward_time=0.080, optim_step_time=0.037, optim0_lr0=1.270e-04, train_time=0.650 +[ALab02] 2023-06-09 14:42:22,696 (trainer:721) INFO: 4epoch:train:2756-3306batch: iter_time=2.196e-04, forward_time=0.055, loss_ctc=7.895, loss=7.895, backward_time=0.078, optim_step_time=0.036, optim0_lr0=1.290e-04, train_time=0.632 +[ALab02] 2023-06-09 14:43:52,888 (trainer:721) INFO: 4epoch:train:3307-3857batch: iter_time=2.181e-04, forward_time=0.057, loss_ctc=7.691, loss=7.691, backward_time=0.080, optim_step_time=0.038, optim0_lr0=1.310e-04, train_time=0.655 +[ALab02] 2023-06-09 14:45:21,916 (trainer:721) INFO: 4epoch:train:3858-4408batch: iter_time=1.985e-04, forward_time=0.056, loss_ctc=7.660, loss=7.660, backward_time=0.079, optim_step_time=0.036, optim0_lr0=1.329e-04, train_time=0.646 +[ALab02] 2023-06-09 14:46:51,064 (trainer:721) INFO: 4epoch:train:4409-4959batch: iter_time=2.016e-04, forward_time=0.056, loss_ctc=7.702, loss=7.702, backward_time=0.079, optim_step_time=0.037, optim0_lr0=1.349e-04, train_time=0.647 +[ALab02] 2023-06-09 14:48:40,834 (trainer:721) INFO: 4epoch:train:4960-5510batch: iter_time=2.020e-04, forward_time=0.081, loss_ctc=7.663, loss=7.663, backward_time=0.085, optim_step_time=0.036, optim0_lr0=1.369e-04, train_time=0.794 +[ALab02] 2023-06-09 14:51:32,729 (trainer:721) INFO: 4epoch:train:5511-6061batch: iter_time=2.178e-04, forward_time=0.155, loss_ctc=7.293, loss=7.293, backward_time=0.102, optim_step_time=0.035, optim0_lr0=1.388e-04, train_time=1.248 +[ALab02] 2023-06-09 14:54:21,176 (trainer:721) INFO: 4epoch:train:6062-6612batch: iter_time=2.053e-04, forward_time=0.150, loss_ctc=7.316, loss=7.316, backward_time=0.101, optim_step_time=0.035, optim0_lr0=1.408e-04, train_time=1.222 +[ALab02] 2023-06-09 14:57:08,359 (trainer:721) INFO: 4epoch:train:6613-7163batch: iter_time=2.098e-04, forward_time=0.149, loss_ctc=7.113, loss=7.113, backward_time=0.100, optim_step_time=0.036, optim0_lr0=1.428e-04, train_time=1.216 +[ALab02] 2023-06-09 15:00:00,055 (trainer:721) INFO: 4epoch:train:7164-7714batch: iter_time=2.125e-04, forward_time=0.155, loss_ctc=6.972, loss=6.972, backward_time=0.102, optim_step_time=0.035, optim0_lr0=1.447e-04, train_time=1.245 +[ALab02] 2023-06-09 15:02:50,450 (trainer:721) INFO: 4epoch:train:7715-8265batch: iter_time=2.046e-04, forward_time=0.152, loss_ctc=6.916, loss=6.916, backward_time=0.102, optim_step_time=0.035, optim0_lr0=1.467e-04, train_time=1.237 +[ALab02] 2023-06-09 15:05:42,005 (trainer:721) INFO: 4epoch:train:8266-8816batch: iter_time=2.039e-04, forward_time=0.154, loss_ctc=6.861, loss=6.861, backward_time=0.102, optim_step_time=0.036, optim0_lr0=1.487e-04, train_time=1.245 +[ALab02] 2023-06-09 15:08:29,570 (trainer:721) INFO: 4epoch:train:8817-9367batch: iter_time=2.055e-04, forward_time=0.151, loss_ctc=6.796, loss=6.796, backward_time=0.099, optim_step_time=0.035, optim0_lr0=1.506e-04, train_time=1.216 +[ALab02] 2023-06-09 15:10:53,758 (trainer:721) INFO: 4epoch:train:9368-9918batch: iter_time=2.056e-04, forward_time=0.123, loss_ctc=6.577, loss=6.577, backward_time=0.093, optim_step_time=0.035, optim0_lr0=1.526e-04, train_time=1.049 +[ALab02] 2023-06-09 15:13:17,222 (trainer:721) INFO: 4epoch:train:9919-10469batch: iter_time=2.078e-04, forward_time=0.121, loss_ctc=6.713, loss=6.713, backward_time=0.094, optim_step_time=0.036, optim0_lr0=1.546e-04, train_time=1.040 +[ALab02] 2023-06-09 15:16:06,228 (trainer:721) INFO: 4epoch:train:10470-11020batch: iter_time=2.005e-04, forward_time=0.152, loss_ctc=6.671, loss=6.671, backward_time=0.101, optim_step_time=0.035, optim0_lr0=1.566e-04, train_time=1.226 +[ALab02] 2023-06-09 15:17:34,882 (trainer:338) INFO: 4epoch results: [train] iter_time=2.380e-04, forward_time=0.103, loss_ctc=7.460, loss=7.460, backward_time=0.090, optim_step_time=0.036, optim0_lr0=1.379e-04, train_time=0.929, time=42 minutes and 42.89 seconds, total_count=44116, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=12.128, cer_ctc=0.200, cer=0.200, loss=12.128, time=56.08 seconds, total_count=1764, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.48 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 15:17:37,327 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 15:17:37,328 (trainer:272) INFO: 5/70epoch started. Estimated time to finish: 1 day, 20 hours and 4 minutes +[ALab02] 2023-06-09 15:20:23,713 (trainer:721) INFO: 5epoch:train:1-551batch: iter_time=0.001, forward_time=0.149, loss_ctc=6.548, loss=6.548, backward_time=0.099, optim_step_time=0.035, optim0_lr0=1.585e-04, train_time=1.208 +[ALab02] 2023-06-09 15:23:12,669 (trainer:721) INFO: 5epoch:train:552-1102batch: iter_time=3.273e-04, forward_time=0.151, loss_ctc=6.381, loss=6.381, backward_time=0.101, optim_step_time=0.034, optim0_lr0=1.605e-04, train_time=1.227 +[ALab02] 2023-06-09 15:26:01,624 (trainer:721) INFO: 5epoch:train:1103-1653batch: iter_time=2.076e-04, forward_time=0.153, loss_ctc=6.306, loss=6.306, backward_time=0.100, optim_step_time=0.035, optim0_lr0=1.625e-04, train_time=1.226 +[ALab02] 2023-06-09 15:28:49,494 (trainer:721) INFO: 5epoch:train:1654-2204batch: iter_time=1.973e-04, forward_time=0.151, loss_ctc=6.314, loss=6.314, backward_time=0.100, optim_step_time=0.034, optim0_lr0=1.645e-04, train_time=1.218 +[ALab02] 2023-06-09 15:31:37,446 (trainer:721) INFO: 5epoch:train:2205-2755batch: iter_time=1.981e-04, forward_time=0.150, loss_ctc=6.124, loss=6.124, backward_time=0.101, optim_step_time=0.034, optim0_lr0=1.664e-04, train_time=1.218 +[ALab02] 2023-06-09 15:34:25,123 (trainer:721) INFO: 5epoch:train:2756-3306batch: iter_time=1.870e-04, forward_time=0.150, loss_ctc=6.368, loss=6.368, backward_time=0.100, optim_step_time=0.034, optim0_lr0=1.684e-04, train_time=1.221 +[ALab02] 2023-06-09 15:37:12,409 (trainer:721) INFO: 5epoch:train:3307-3857batch: iter_time=2.519e-04, forward_time=0.150, loss_ctc=6.147, loss=6.147, backward_time=0.100, optim_step_time=0.035, optim0_lr0=1.704e-04, train_time=1.213 +[ALab02] 2023-06-09 15:39:59,037 (trainer:721) INFO: 5epoch:train:3858-4408batch: iter_time=1.955e-04, forward_time=0.149, loss_ctc=5.958, loss=5.958, backward_time=0.100, optim_step_time=0.034, optim0_lr0=1.723e-04, train_time=1.209 +[ALab02] 2023-06-09 15:42:45,485 (trainer:721) INFO: 5epoch:train:4409-4959batch: iter_time=1.864e-04, forward_time=0.150, loss_ctc=5.941, loss=5.941, backward_time=0.099, optim_step_time=0.034, optim0_lr0=1.743e-04, train_time=1.208 +[ALab02] 2023-06-09 15:45:26,584 (trainer:721) INFO: 5epoch:train:4960-5510batch: iter_time=1.877e-04, forward_time=0.142, loss_ctc=5.754, loss=5.754, backward_time=0.099, optim_step_time=0.034, optim0_lr0=1.763e-04, train_time=1.171 +[ALab02] 2023-06-09 15:47:36,525 (trainer:721) INFO: 5epoch:train:5511-6061batch: iter_time=1.916e-04, forward_time=0.106, loss_ctc=5.806, loss=5.806, backward_time=0.089, optim_step_time=0.034, optim0_lr0=1.782e-04, train_time=0.943 +[ALab02] 2023-06-09 15:50:26,401 (trainer:721) INFO: 5epoch:train:6062-6612batch: iter_time=1.916e-04, forward_time=0.154, loss_ctc=5.832, loss=5.832, backward_time=0.100, optim_step_time=0.034, optim0_lr0=1.802e-04, train_time=1.232 +[ALab02] 2023-06-09 15:53:15,832 (trainer:721) INFO: 5epoch:train:6613-7163batch: iter_time=1.948e-04, forward_time=0.153, loss_ctc=5.670, loss=5.670, backward_time=0.101, optim_step_time=0.036, optim0_lr0=1.822e-04, train_time=1.230 +[ALab02] 2023-06-09 15:56:04,221 (trainer:721) INFO: 5epoch:train:7164-7714batch: iter_time=2.142e-04, forward_time=0.150, loss_ctc=5.696, loss=5.696, backward_time=0.101, optim_step_time=0.036, optim0_lr0=1.841e-04, train_time=1.223 +[ALab02] 2023-06-09 15:58:52,796 (trainer:721) INFO: 5epoch:train:7715-8265batch: iter_time=1.986e-04, forward_time=0.152, loss_ctc=5.660, loss=5.660, backward_time=0.100, optim_step_time=0.036, optim0_lr0=1.861e-04, train_time=1.223 +[ALab02] 2023-06-09 16:01:39,910 (trainer:721) INFO: 5epoch:train:8266-8816batch: iter_time=1.990e-04, forward_time=0.151, loss_ctc=5.615, loss=5.615, backward_time=0.099, optim_step_time=0.035, optim0_lr0=1.881e-04, train_time=1.213 +[ALab02] 2023-06-09 16:04:29,400 (trainer:721) INFO: 5epoch:train:8817-9367batch: iter_time=1.950e-04, forward_time=0.153, loss_ctc=5.614, loss=5.614, backward_time=0.101, optim_step_time=0.035, optim0_lr0=1.900e-04, train_time=1.231 +[ALab02] 2023-06-09 16:07:16,191 (trainer:721) INFO: 5epoch:train:9368-9918batch: iter_time=1.980e-04, forward_time=0.149, loss_ctc=5.466, loss=5.466, backward_time=0.100, optim_step_time=0.036, optim0_lr0=1.920e-04, train_time=1.209 +[ALab02] 2023-06-09 16:10:04,930 (trainer:721) INFO: 5epoch:train:9919-10469batch: iter_time=2.008e-04, forward_time=0.152, loss_ctc=5.375, loss=5.375, backward_time=0.101, optim_step_time=0.035, optim0_lr0=1.940e-04, train_time=1.225 +[ALab02] 2023-06-09 16:12:54,861 (trainer:721) INFO: 5epoch:train:10470-11020batch: iter_time=2.040e-04, forward_time=0.153, loss_ctc=5.363, loss=5.363, backward_time=0.101, optim_step_time=0.035, optim0_lr0=1.959e-04, train_time=1.233 +[ALab02] 2023-06-09 16:14:25,332 (trainer:338) INFO: 5epoch results: [train] iter_time=2.526e-04, forward_time=0.148, loss_ctc=5.897, loss=5.897, backward_time=0.100, optim_step_time=0.035, optim0_lr0=1.773e-04, train_time=1.204, time=55 minutes and 20.63 seconds, total_count=55145, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=9.260, cer_ctc=0.153, cer=0.153, loss=9.260, time=55.95 seconds, total_count=2205, gpu_max_cached_mem_GB=10.861, [att_plot] time=31.43 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 16:14:27,978 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 16:14:27,979 (trainer:272) INFO: 6/70epoch started. Estimated time to finish: 1 day, 23 hours and 2 minutes +[ALab02] 2023-06-09 16:17:17,178 (trainer:721) INFO: 6epoch:train:1-551batch: iter_time=9.325e-04, forward_time=0.152, loss_ctc=5.367, loss=5.367, backward_time=0.101, optim_step_time=0.036, optim0_lr0=1.979e-04, train_time=1.230 +[ALab02] 2023-06-09 16:20:04,355 (trainer:721) INFO: 6epoch:train:552-1102batch: iter_time=2.293e-04, forward_time=0.150, loss_ctc=5.245, loss=5.245, backward_time=0.100, optim_step_time=0.035, optim0_lr0=1.999e-04, train_time=1.211 +[ALab02] 2023-06-09 16:22:18,037 (trainer:721) INFO: 6epoch:train:1103-1653batch: iter_time=2.027e-04, forward_time=0.111, loss_ctc=5.124, loss=5.124, backward_time=0.090, optim_step_time=0.036, optim0_lr0=2.019e-04, train_time=0.971 +[ALab02] 2023-06-09 16:25:05,415 (trainer:721) INFO: 6epoch:train:1654-2204batch: iter_time=2.165e-04, forward_time=0.150, loss_ctc=5.303, loss=5.303, backward_time=0.100, optim_step_time=0.036, optim0_lr0=2.038e-04, train_time=1.215 +[ALab02] 2023-06-09 16:27:54,282 (trainer:721) INFO: 6epoch:train:2205-2755batch: iter_time=2.067e-04, forward_time=0.151, loss_ctc=5.105, loss=5.105, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.058e-04, train_time=1.226 +[ALab02] 2023-06-09 16:30:42,840 (trainer:721) INFO: 6epoch:train:2756-3306batch: iter_time=2.074e-04, forward_time=0.152, loss_ctc=4.942, loss=4.942, backward_time=0.100, optim_step_time=0.036, optim0_lr0=2.078e-04, train_time=1.223 +[ALab02] 2023-06-09 16:33:30,243 (trainer:721) INFO: 6epoch:train:3307-3857batch: iter_time=2.017e-04, forward_time=0.150, loss_ctc=5.024, loss=5.024, backward_time=0.099, optim_step_time=0.035, optim0_lr0=2.097e-04, train_time=1.216 +[ALab02] 2023-06-09 16:36:16,263 (trainer:721) INFO: 6epoch:train:3858-4408batch: iter_time=1.877e-04, forward_time=0.149, loss_ctc=4.954, loss=4.954, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.117e-04, train_time=1.205 +[ALab02] 2023-06-09 16:39:10,489 (trainer:721) INFO: 6epoch:train:4409-4959batch: iter_time=2.319e-04, forward_time=0.156, loss_ctc=4.949, loss=4.949, backward_time=0.105, optim_step_time=0.038, optim0_lr0=2.137e-04, train_time=1.266 +[ALab02] 2023-06-09 16:41:59,949 (trainer:721) INFO: 6epoch:train:4960-5510batch: iter_time=2.089e-04, forward_time=0.153, loss_ctc=4.956, loss=4.956, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.156e-04, train_time=1.230 +[ALab02] 2023-06-09 16:44:47,196 (trainer:721) INFO: 6epoch:train:5511-6061batch: iter_time=2.045e-04, forward_time=0.149, loss_ctc=4.850, loss=4.850, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.176e-04, train_time=1.213 +[ALab02] 2023-06-09 16:47:37,747 (trainer:721) INFO: 6epoch:train:6062-6612batch: iter_time=2.072e-04, forward_time=0.154, loss_ctc=4.749, loss=4.749, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.196e-04, train_time=1.238 +[ALab02] 2023-06-09 16:50:25,621 (trainer:721) INFO: 6epoch:train:6613-7163batch: iter_time=2.070e-04, forward_time=0.150, loss_ctc=4.932, loss=4.932, backward_time=0.101, optim_step_time=0.037, optim0_lr0=2.215e-04, train_time=1.217 +[ALab02] 2023-06-09 16:53:14,762 (trainer:721) INFO: 6epoch:train:7164-7714batch: iter_time=2.628e-04, forward_time=0.151, loss_ctc=4.918, loss=4.918, backward_time=0.101, optim_step_time=0.036, optim0_lr0=2.235e-04, train_time=1.229 +[ALab02] 2023-06-09 16:55:42,492 (trainer:721) INFO: 6epoch:train:7715-8265batch: iter_time=1.923e-04, forward_time=0.127, loss_ctc=4.829, loss=4.829, backward_time=0.093, optim_step_time=0.035, optim0_lr0=2.255e-04, train_time=1.073 +[ALab02] 2023-06-09 16:58:11,862 (trainer:721) INFO: 6epoch:train:8266-8816batch: iter_time=1.815e-04, forward_time=0.129, loss_ctc=4.738, loss=4.738, backward_time=0.094, optim_step_time=0.034, optim0_lr0=2.275e-04, train_time=1.083 +[ALab02] 2023-06-09 17:01:04,837 (trainer:721) INFO: 6epoch:train:8817-9367batch: iter_time=1.958e-04, forward_time=0.155, loss_ctc=4.653, loss=4.653, backward_time=0.103, optim_step_time=0.036, optim0_lr0=2.294e-04, train_time=1.255 +[ALab02] 2023-06-09 17:03:51,242 (trainer:721) INFO: 6epoch:train:9368-9918batch: iter_time=1.915e-04, forward_time=0.147, loss_ctc=4.769, loss=4.769, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.314e-04, train_time=1.209 +[ALab02] 2023-06-09 17:06:41,856 (trainer:721) INFO: 6epoch:train:9919-10469batch: iter_time=1.901e-04, forward_time=0.153, loss_ctc=4.529, loss=4.529, backward_time=0.102, optim_step_time=0.035, optim0_lr0=2.334e-04, train_time=1.237 +[ALab02] 2023-06-09 17:09:30,448 (trainer:721) INFO: 6epoch:train:10470-11020batch: iter_time=2.014e-04, forward_time=0.150, loss_ctc=4.605, loss=4.605, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.353e-04, train_time=1.225 +[ALab02] 2023-06-09 17:10:58,788 (trainer:338) INFO: 6epoch results: [train] iter_time=2.429e-04, forward_time=0.147, loss_ctc=4.925, loss=4.925, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.166e-04, train_time=1.198, time=55 minutes and 5.29 seconds, total_count=66174, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=7.917, cer_ctc=0.126, cer=0.126, loss=7.917, time=56.03 seconds, total_count=2646, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.49 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 17:11:01,431 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 17:11:01,431 (trainer:272) INFO: 7/70epoch started. Estimated time to finish: 2 days, 38 minutes and 58.17 seconds +[ALab02] 2023-06-09 17:13:48,431 (trainer:721) INFO: 7epoch:train:1-551batch: iter_time=9.498e-04, forward_time=0.149, loss_ctc=4.456, loss=4.456, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.373e-04, train_time=1.211 +[ALab02] 2023-06-09 17:16:36,739 (trainer:721) INFO: 7epoch:train:552-1102batch: iter_time=3.136e-04, forward_time=0.152, loss_ctc=4.562, loss=4.562, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.393e-04, train_time=1.225 +[ALab02] 2023-06-09 17:19:25,895 (trainer:721) INFO: 7epoch:train:1103-1653batch: iter_time=3.097e-04, forward_time=0.152, loss_ctc=4.492, loss=4.492, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.413e-04, train_time=1.225 +[ALab02] 2023-06-09 17:22:14,826 (trainer:721) INFO: 7epoch:train:1654-2204batch: iter_time=2.032e-04, forward_time=0.152, loss_ctc=4.536, loss=4.536, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.432e-04, train_time=1.226 +[ALab02] 2023-06-09 17:25:01,772 (trainer:721) INFO: 7epoch:train:2205-2755batch: iter_time=2.017e-04, forward_time=0.149, loss_ctc=4.369, loss=4.369, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.452e-04, train_time=1.211 +[ALab02] 2023-06-09 17:27:49,306 (trainer:721) INFO: 7epoch:train:2756-3306batch: iter_time=1.984e-04, forward_time=0.149, loss_ctc=4.609, loss=4.609, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.472e-04, train_time=1.218 +[ALab02] 2023-06-09 17:30:23,156 (trainer:721) INFO: 7epoch:train:3307-3857batch: iter_time=1.982e-04, forward_time=0.133, loss_ctc=4.480, loss=4.480, backward_time=0.097, optim_step_time=0.035, optim0_lr0=2.491e-04, train_time=1.117 +[ALab02] 2023-06-09 17:32:52,832 (trainer:721) INFO: 7epoch:train:3858-4408batch: iter_time=1.945e-04, forward_time=0.129, loss_ctc=4.477, loss=4.477, backward_time=0.095, optim_step_time=0.035, optim0_lr0=2.511e-04, train_time=1.085 +[ALab02] 2023-06-09 17:35:39,233 (trainer:721) INFO: 7epoch:train:4409-4959batch: iter_time=1.948e-04, forward_time=0.149, loss_ctc=4.421, loss=4.421, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.531e-04, train_time=1.207 +[ALab02] 2023-06-09 17:38:26,048 (trainer:721) INFO: 7epoch:train:4960-5510batch: iter_time=1.892e-04, forward_time=0.149, loss_ctc=4.377, loss=4.377, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.550e-04, train_time=1.210 +[ALab02] 2023-06-09 17:41:13,514 (trainer:721) INFO: 7epoch:train:5511-6061batch: iter_time=1.881e-04, forward_time=0.149, loss_ctc=4.304, loss=4.304, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.570e-04, train_time=1.216 +[ALab02] 2023-06-09 17:44:00,401 (trainer:721) INFO: 7epoch:train:6062-6612batch: iter_time=1.968e-04, forward_time=0.150, loss_ctc=4.260, loss=4.260, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.590e-04, train_time=1.211 +[ALab02] 2023-06-09 17:46:47,707 (trainer:721) INFO: 7epoch:train:6613-7163batch: iter_time=1.955e-04, forward_time=0.149, loss_ctc=4.314, loss=4.314, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.609e-04, train_time=1.217 +[ALab02] 2023-06-09 17:49:36,495 (trainer:721) INFO: 7epoch:train:7164-7714batch: iter_time=2.102e-04, forward_time=0.152, loss_ctc=4.278, loss=4.278, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.629e-04, train_time=1.223 +[ALab02] 2023-06-09 17:52:25,492 (trainer:721) INFO: 7epoch:train:7715-8265batch: iter_time=2.160e-04, forward_time=0.153, loss_ctc=4.232, loss=4.232, backward_time=0.099, optim_step_time=0.035, optim0_lr0=2.649e-04, train_time=1.226 +[ALab02] 2023-06-09 17:55:13,267 (trainer:721) INFO: 7epoch:train:8266-8816batch: iter_time=2.128e-04, forward_time=0.151, loss_ctc=4.174, loss=4.174, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.668e-04, train_time=1.218 +[ALab02] 2023-06-09 17:58:00,769 (trainer:721) INFO: 7epoch:train:8817-9367batch: iter_time=2.113e-04, forward_time=0.150, loss_ctc=4.274, loss=4.274, backward_time=0.101, optim_step_time=0.036, optim0_lr0=2.688e-04, train_time=1.216 +[ALab02] 2023-06-09 18:00:47,853 (trainer:721) INFO: 7epoch:train:9368-9918batch: iter_time=2.113e-04, forward_time=0.150, loss_ctc=4.368, loss=4.368, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.708e-04, train_time=1.214 +[ALab02] 2023-06-09 18:03:39,424 (trainer:721) INFO: 7epoch:train:9919-10469batch: iter_time=2.135e-04, forward_time=0.155, loss_ctc=4.262, loss=4.262, backward_time=0.102, optim_step_time=0.036, optim0_lr0=2.727e-04, train_time=1.244 +[ALab02] 2023-06-09 18:05:55,032 (trainer:721) INFO: 7epoch:train:10470-11020batch: iter_time=2.055e-04, forward_time=0.112, loss_ctc=4.180, loss=4.180, backward_time=0.091, optim_step_time=0.035, optim0_lr0=2.747e-04, train_time=0.985 +[ALab02] 2023-06-09 18:07:22,066 (trainer:338) INFO: 7epoch results: [train] iter_time=2.506e-04, forward_time=0.147, loss_ctc=4.371, loss=4.371, backward_time=0.099, optim_step_time=0.035, optim0_lr0=2.560e-04, train_time=1.195, time=54 minutes and 56.61 seconds, total_count=77203, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=7.090, cer_ctc=0.112, cer=0.112, loss=7.090, time=54.67 seconds, total_count=3087, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.36 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 18:07:24,815 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 18:07:24,816 (trainer:272) INFO: 8/70epoch started. Estimated time to finish: 2 days, 1 hour and 30 minutes +[ALab02] 2023-06-09 18:10:16,064 (trainer:721) INFO: 8epoch:train:1-551batch: iter_time=7.036e-04, forward_time=0.152, loss_ctc=4.098, loss=4.098, backward_time=0.102, optim_step_time=0.036, optim0_lr0=2.767e-04, train_time=1.244 +[ALab02] 2023-06-09 18:13:04,104 (trainer:721) INFO: 8epoch:train:552-1102batch: iter_time=2.154e-04, forward_time=0.150, loss_ctc=4.110, loss=4.110, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.787e-04, train_time=1.218 +[ALab02] 2023-06-09 18:15:52,971 (trainer:721) INFO: 8epoch:train:1103-1653batch: iter_time=2.226e-04, forward_time=0.152, loss_ctc=4.138, loss=4.138, backward_time=0.100, optim_step_time=0.036, optim0_lr0=2.806e-04, train_time=1.226 +[ALab02] 2023-06-09 18:18:40,783 (trainer:721) INFO: 8epoch:train:1654-2204batch: iter_time=2.120e-04, forward_time=0.151, loss_ctc=4.079, loss=4.079, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.826e-04, train_time=1.218 +[ALab02] 2023-06-09 18:21:28,438 (trainer:721) INFO: 8epoch:train:2205-2755batch: iter_time=1.906e-04, forward_time=0.150, loss_ctc=4.080, loss=4.080, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.846e-04, train_time=1.217 +[ALab02] 2023-06-09 18:24:23,024 (trainer:721) INFO: 8epoch:train:2756-3306batch: iter_time=2.376e-04, forward_time=0.156, loss_ctc=4.014, loss=4.014, backward_time=0.106, optim_step_time=0.040, optim0_lr0=2.865e-04, train_time=1.266 +[ALab02] 2023-06-09 18:27:11,139 (trainer:721) INFO: 8epoch:train:3307-3857batch: iter_time=1.978e-04, forward_time=0.150, loss_ctc=4.088, loss=4.088, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.885e-04, train_time=1.222 +[ALab02] 2023-06-09 18:30:01,771 (trainer:721) INFO: 8epoch:train:3858-4408batch: iter_time=2.043e-04, forward_time=0.155, loss_ctc=4.029, loss=4.029, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.905e-04, train_time=1.238 +[ALab02] 2023-06-09 18:32:50,294 (trainer:721) INFO: 8epoch:train:4409-4959batch: iter_time=2.127e-04, forward_time=0.151, loss_ctc=3.956, loss=3.956, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.924e-04, train_time=1.224 +[ALab02] 2023-06-09 18:35:39,840 (trainer:721) INFO: 8epoch:train:4960-5510batch: iter_time=1.958e-04, forward_time=0.153, loss_ctc=3.844, loss=3.844, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.944e-04, train_time=1.230 +[ALab02] 2023-06-09 18:38:29,247 (trainer:721) INFO: 8epoch:train:5511-6061batch: iter_time=2.044e-04, forward_time=0.153, loss_ctc=3.731, loss=3.731, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.964e-04, train_time=1.230 +[ALab02] 2023-06-09 18:40:44,120 (trainer:721) INFO: 8epoch:train:6062-6612batch: iter_time=1.970e-04, forward_time=0.111, loss_ctc=3.960, loss=3.960, backward_time=0.091, optim_step_time=0.035, optim0_lr0=2.984e-04, train_time=0.978 +[ALab02] 2023-06-09 18:43:33,538 (trainer:721) INFO: 8epoch:train:6613-7163batch: iter_time=1.929e-04, forward_time=0.151, loss_ctc=3.999, loss=3.999, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.003e-04, train_time=1.229 +[ALab02] 2023-06-09 18:46:23,985 (trainer:721) INFO: 8epoch:train:7164-7714batch: iter_time=2.112e-04, forward_time=0.152, loss_ctc=4.008, loss=4.008, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.023e-04, train_time=1.239 +[ALab02] 2023-06-09 18:49:13,124 (trainer:721) INFO: 8epoch:train:7715-8265batch: iter_time=2.027e-04, forward_time=0.153, loss_ctc=3.924, loss=3.924, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.042e-04, train_time=1.226 +[ALab02] 2023-06-09 18:52:03,454 (trainer:721) INFO: 8epoch:train:8266-8816batch: iter_time=1.980e-04, forward_time=0.154, loss_ctc=4.066, loss=4.066, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.062e-04, train_time=1.236 +[ALab02] 2023-06-09 18:54:51,440 (trainer:721) INFO: 8epoch:train:8817-9367batch: iter_time=1.989e-04, forward_time=0.152, loss_ctc=4.057, loss=4.057, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.082e-04, train_time=1.219 +[ALab02] 2023-06-09 18:57:39,591 (trainer:721) INFO: 8epoch:train:9368-9918batch: iter_time=2.075e-04, forward_time=0.151, loss_ctc=4.170, loss=4.170, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.102e-04, train_time=1.221 +[ALab02] 2023-06-09 19:00:28,303 (trainer:721) INFO: 8epoch:train:9919-10469batch: iter_time=2.050e-04, forward_time=0.150, loss_ctc=4.054, loss=4.054, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.121e-04, train_time=1.226 +[ALab02] 2023-06-09 19:03:16,514 (trainer:721) INFO: 8epoch:train:10470-11020batch: iter_time=1.936e-04, forward_time=0.151, loss_ctc=3.921, loss=3.921, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.141e-04, train_time=1.220 +[ALab02] 2023-06-09 19:04:45,086 (trainer:338) INFO: 8epoch results: [train] iter_time=2.301e-04, forward_time=0.150, loss_ctc=4.015, loss=4.015, backward_time=0.101, optim_step_time=0.036, optim0_lr0=2.954e-04, train_time=1.216, time=55 minutes and 55.19 seconds, total_count=88232, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=6.708, cer_ctc=0.103, cer=0.103, loss=6.708, time=55.47 seconds, total_count=3528, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.61 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 19:04:48,185 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 19:04:48,186 (trainer:272) INFO: 9/70epoch started. Estimated time to finish: 2 days, 2 hours and 2 minutes +[ALab02] 2023-06-09 19:07:38,271 (trainer:721) INFO: 9epoch:train:1-551batch: iter_time=0.001, forward_time=0.152, loss_ctc=3.864, loss=3.864, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.161e-04, train_time=1.235 +[ALab02] 2023-06-09 19:10:25,740 (trainer:721) INFO: 9epoch:train:552-1102batch: iter_time=2.235e-04, forward_time=0.149, loss_ctc=3.945, loss=3.945, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.181e-04, train_time=1.217 +[ALab02] 2023-06-09 19:13:15,497 (trainer:721) INFO: 9epoch:train:1103-1653batch: iter_time=2.070e-04, forward_time=0.153, loss_ctc=3.848, loss=3.848, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.200e-04, train_time=1.230 +[ALab02] 2023-06-09 19:15:32,219 (trainer:721) INFO: 9epoch:train:1654-2204batch: iter_time=2.172e-04, forward_time=0.114, loss_ctc=3.689, loss=3.689, backward_time=0.091, optim_step_time=0.036, optim0_lr0=3.220e-04, train_time=0.993 +[ALab02] 2023-06-09 19:18:23,400 (trainer:721) INFO: 9epoch:train:2205-2755batch: iter_time=2.090e-04, forward_time=0.152, loss_ctc=3.772, loss=3.772, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.240e-04, train_time=1.240 +[ALab02] 2023-06-09 19:21:13,174 (trainer:721) INFO: 9epoch:train:2756-3306batch: iter_time=2.112e-04, forward_time=0.151, loss_ctc=3.759, loss=3.759, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.259e-04, train_time=1.234 +[ALab02] 2023-06-09 19:24:01,247 (trainer:721) INFO: 9epoch:train:3307-3857batch: iter_time=2.126e-04, forward_time=0.149, loss_ctc=3.778, loss=3.778, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.279e-04, train_time=1.221 +[ALab02] 2023-06-09 19:26:51,417 (trainer:721) INFO: 9epoch:train:3858-4408batch: iter_time=2.039e-04, forward_time=0.151, loss_ctc=3.694, loss=3.694, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.299e-04, train_time=1.235 +[ALab02] 2023-06-09 19:29:39,953 (trainer:721) INFO: 9epoch:train:4409-4959batch: iter_time=2.163e-04, forward_time=0.151, loss_ctc=3.695, loss=3.695, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.318e-04, train_time=1.223 +[ALab02] 2023-06-09 19:32:27,973 (trainer:721) INFO: 9epoch:train:4960-5510batch: iter_time=2.118e-04, forward_time=0.150, loss_ctc=3.807, loss=3.807, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.338e-04, train_time=1.222 +[ALab02] 2023-06-09 19:35:17,444 (trainer:721) INFO: 9epoch:train:5511-6061batch: iter_time=2.079e-04, forward_time=0.152, loss_ctc=3.830, loss=3.830, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.358e-04, train_time=1.228 +[ALab02] 2023-06-09 19:38:07,030 (trainer:721) INFO: 9epoch:train:6062-6612batch: iter_time=2.088e-04, forward_time=0.152, loss_ctc=3.788, loss=3.788, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.377e-04, train_time=1.231 +[ALab02] 2023-06-09 19:40:53,447 (trainer:721) INFO: 9epoch:train:6613-7163batch: iter_time=2.135e-04, forward_time=0.149, loss_ctc=3.706, loss=3.706, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.397e-04, train_time=1.209 +[ALab02] 2023-06-09 19:43:40,168 (trainer:721) INFO: 9epoch:train:7164-7714batch: iter_time=2.043e-04, forward_time=0.149, loss_ctc=3.691, loss=3.691, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.417e-04, train_time=1.212 +[ALab02] 2023-06-09 19:46:30,839 (trainer:721) INFO: 9epoch:train:7715-8265batch: iter_time=2.060e-04, forward_time=0.153, loss_ctc=3.597, loss=3.597, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.436e-04, train_time=1.237 +[ALab02] 2023-06-09 19:49:20,154 (trainer:721) INFO: 9epoch:train:8266-8816batch: iter_time=2.114e-04, forward_time=0.150, loss_ctc=3.694, loss=3.694, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.456e-04, train_time=1.228 +[ALab02] 2023-06-09 19:51:35,629 (trainer:721) INFO: 9epoch:train:8817-9367batch: iter_time=2.150e-04, forward_time=0.113, loss_ctc=3.606, loss=3.606, backward_time=0.091, optim_step_time=0.036, optim0_lr0=3.476e-04, train_time=0.981 +[ALab02] 2023-06-09 19:54:23,822 (trainer:721) INFO: 9epoch:train:9368-9918batch: iter_time=2.047e-04, forward_time=0.149, loss_ctc=3.522, loss=3.522, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.495e-04, train_time=1.222 +[ALab02] 2023-06-09 19:57:13,992 (trainer:721) INFO: 9epoch:train:9919-10469batch: iter_time=2.426e-04, forward_time=0.153, loss_ctc=3.597, loss=3.597, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.515e-04, train_time=1.235 +[ALab02] 2023-06-09 20:00:04,063 (trainer:721) INFO: 9epoch:train:10470-11020batch: iter_time=2.145e-04, forward_time=0.152, loss_ctc=3.516, loss=3.516, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.535e-04, train_time=1.234 +[ALab02] 2023-06-09 20:01:33,820 (trainer:338) INFO: 9epoch results: [train] iter_time=2.650e-04, forward_time=0.147, loss_ctc=3.720, loss=3.720, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.348e-04, train_time=1.203, time=55 minutes and 18.99 seconds, total_count=99261, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=6.178, cer_ctc=0.096, cer=0.096, loss=6.178, time=56.18 seconds, total_count=3969, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.45 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 20:01:36,378 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 20:01:36,378 (trainer:272) INFO: 10/70epoch started. Estimated time to finish: 2 days, 2 hours and 10 minutes +[ALab02] 2023-06-09 20:04:24,156 (trainer:721) INFO: 10epoch:train:1-551batch: iter_time=0.001, forward_time=0.150, loss_ctc=3.500, loss=3.500, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.555e-04, train_time=1.218 +[ALab02] 2023-06-09 20:07:11,565 (trainer:721) INFO: 10epoch:train:552-1102batch: iter_time=3.018e-04, forward_time=0.149, loss_ctc=3.390, loss=3.390, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.574e-04, train_time=1.217 +[ALab02] 2023-06-09 20:10:04,828 (trainer:721) INFO: 10epoch:train:1103-1653batch: iter_time=2.391e-04, forward_time=0.154, loss_ctc=3.490, loss=3.490, backward_time=0.105, optim_step_time=0.038, optim0_lr0=3.594e-04, train_time=1.257 +[ALab02] 2023-06-09 20:12:57,552 (trainer:721) INFO: 10epoch:train:1654-2204batch: iter_time=2.264e-04, forward_time=0.156, loss_ctc=3.445, loss=3.445, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.614e-04, train_time=1.253 +[ALab02] 2023-06-09 20:15:47,469 (trainer:721) INFO: 10epoch:train:2205-2755batch: iter_time=2.220e-04, forward_time=0.152, loss_ctc=3.426, loss=3.426, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.633e-04, train_time=1.234 +[ALab02] 2023-06-09 20:18:40,798 (trainer:721) INFO: 10epoch:train:2756-3306batch: iter_time=2.257e-04, forward_time=0.156, loss_ctc=3.315, loss=3.315, backward_time=0.104, optim_step_time=0.037, optim0_lr0=3.653e-04, train_time=1.258 +[ALab02] 2023-06-09 20:21:29,886 (trainer:721) INFO: 10epoch:train:3307-3857batch: iter_time=2.096e-04, forward_time=0.152, loss_ctc=3.339, loss=3.339, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.673e-04, train_time=1.226 +[ALab02] 2023-06-09 20:24:19,183 (trainer:721) INFO: 10epoch:train:3858-4408batch: iter_time=2.324e-04, forward_time=0.150, loss_ctc=3.279, loss=3.279, backward_time=0.104, optim_step_time=0.038, optim0_lr0=3.693e-04, train_time=1.229 +[ALab02] 2023-06-09 20:26:42,137 (trainer:721) INFO: 10epoch:train:4409-4959batch: iter_time=2.275e-04, forward_time=0.120, loss_ctc=3.426, loss=3.426, backward_time=0.096, optim_step_time=0.036, optim0_lr0=3.712e-04, train_time=1.038 +[ALab02] 2023-06-09 20:29:31,871 (trainer:721) INFO: 10epoch:train:4960-5510batch: iter_time=2.136e-04, forward_time=0.152, loss_ctc=3.365, loss=3.365, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.732e-04, train_time=1.231 +[ALab02] 2023-06-09 20:32:25,953 (trainer:721) INFO: 10epoch:train:5511-6061batch: iter_time=2.351e-04, forward_time=0.155, loss_ctc=3.343, loss=3.343, backward_time=0.106, optim_step_time=0.039, optim0_lr0=3.752e-04, train_time=1.264 +[ALab02] 2023-06-09 20:35:17,631 (trainer:721) INFO: 10epoch:train:6062-6612batch: iter_time=2.187e-04, forward_time=0.153, loss_ctc=3.289, loss=3.289, backward_time=0.104, optim_step_time=0.037, optim0_lr0=3.771e-04, train_time=1.245 +[ALab02] 2023-06-09 20:38:10,444 (trainer:721) INFO: 10epoch:train:6613-7163batch: iter_time=2.327e-04, forward_time=0.154, loss_ctc=3.250, loss=3.250, backward_time=0.106, optim_step_time=0.037, optim0_lr0=3.791e-04, train_time=1.254 +[ALab02] 2023-06-09 20:41:02,635 (trainer:721) INFO: 10epoch:train:7164-7714batch: iter_time=2.325e-04, forward_time=0.155, loss_ctc=3.308, loss=3.308, backward_time=0.104, optim_step_time=0.037, optim0_lr0=3.811e-04, train_time=1.250 +[ALab02] 2023-06-09 20:43:54,205 (trainer:721) INFO: 10epoch:train:7715-8265batch: iter_time=2.198e-04, forward_time=0.154, loss_ctc=3.208, loss=3.208, backward_time=0.103, optim_step_time=0.038, optim0_lr0=3.830e-04, train_time=1.245 +[ALab02] 2023-06-09 20:46:49,568 (trainer:721) INFO: 10epoch:train:8266-8816batch: iter_time=2.415e-04, forward_time=0.158, loss_ctc=3.224, loss=3.224, backward_time=0.105, optim_step_time=0.037, optim0_lr0=3.850e-04, train_time=1.272 +[ALab02] 2023-06-09 20:49:39,975 (trainer:721) INFO: 10epoch:train:8817-9367batch: iter_time=2.141e-04, forward_time=0.152, loss_ctc=3.350, loss=3.350, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.870e-04, train_time=1.238 +[ALab02] 2023-06-09 20:52:33,775 (trainer:721) INFO: 10epoch:train:9368-9918batch: iter_time=2.337e-04, forward_time=0.157, loss_ctc=3.341, loss=3.341, backward_time=0.104, optim_step_time=0.039, optim0_lr0=3.889e-04, train_time=1.260 +[ALab02] 2023-06-09 20:55:27,280 (trainer:721) INFO: 10epoch:train:9919-10469batch: iter_time=2.238e-04, forward_time=0.156, loss_ctc=3.209, loss=3.209, backward_time=0.105, optim_step_time=0.036, optim0_lr0=3.909e-04, train_time=1.259 +[ALab02] 2023-06-09 20:58:15,831 (trainer:721) INFO: 10epoch:train:10470-11020batch: iter_time=2.114e-04, forward_time=0.152, loss_ctc=3.258, loss=3.258, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.929e-04, train_time=1.223 +[ALab02] 2023-06-09 20:59:42,186 (trainer:338) INFO: 10epoch results: [train] iter_time=2.704e-04, forward_time=0.152, loss_ctc=3.337, loss=3.337, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.742e-04, train_time=1.234, time=56 minutes and 42.57 seconds, total_count=110290, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=5.826, cer_ctc=0.091, cer=0.091, loss=5.826, time=52.15 seconds, total_count=4410, gpu_max_cached_mem_GB=10.861, [att_plot] time=31.09 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 20:59:44,880 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 20:59:44,880 (trainer:272) INFO: 11/70epoch started. Estimated time to finish: 2 days, 2 hours and 14 minutes +[ALab02] 2023-06-09 21:02:31,782 (trainer:721) INFO: 11epoch:train:1-551batch: iter_time=9.355e-04, forward_time=0.149, loss_ctc=3.111, loss=3.111, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.949e-04, train_time=1.214 +[ALab02] 2023-06-09 21:05:23,234 (trainer:721) INFO: 11epoch:train:552-1102batch: iter_time=2.296e-04, forward_time=0.154, loss_ctc=3.171, loss=3.171, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.968e-04, train_time=1.242 +[ALab02] 2023-06-09 21:08:13,348 (trainer:721) INFO: 11epoch:train:1103-1653batch: iter_time=2.391e-04, forward_time=0.153, loss_ctc=3.170, loss=3.170, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.988e-04, train_time=1.235 +[ALab02] 2023-06-09 21:11:04,448 (trainer:721) INFO: 11epoch:train:1654-2204batch: iter_time=2.154e-04, forward_time=0.152, loss_ctc=3.196, loss=3.196, backward_time=0.104, optim_step_time=0.036, optim0_lr0=4.008e-04, train_time=1.241 +[ALab02] 2023-06-09 21:14:02,022 (trainer:721) INFO: 11epoch:train:2205-2755batch: iter_time=3.307e-04, forward_time=0.151, loss_ctc=3.129, loss=3.129, backward_time=0.118, optim_step_time=0.052, optim0_lr0=4.027e-04, train_time=1.288 +[ALab02] 2023-06-09 21:16:38,425 (trainer:721) INFO: 11epoch:train:2756-3306batch: iter_time=3.057e-04, forward_time=0.126, loss_ctc=3.060, loss=3.060, backward_time=0.111, optim_step_time=0.047, optim0_lr0=4.047e-04, train_time=1.136 +[ALab02] 2023-06-09 21:19:12,625 (trainer:721) INFO: 11epoch:train:3307-3857batch: iter_time=3.092e-04, forward_time=0.123, loss_ctc=3.061, loss=3.061, backward_time=0.110, optim_step_time=0.046, optim0_lr0=4.067e-04, train_time=1.119 +[ALab02] 2023-06-09 21:21:37,697 (trainer:721) INFO: 11epoch:train:3858-4408batch: iter_time=3.113e-04, forward_time=0.113, loss_ctc=3.125, loss=3.125, backward_time=0.107, optim_step_time=0.047, optim0_lr0=4.086e-04, train_time=1.053 +[ALab02] 2023-06-09 21:24:03,621 (trainer:721) INFO: 11epoch:train:4409-4959batch: iter_time=2.954e-04, forward_time=0.114, loss_ctc=3.029, loss=3.029, backward_time=0.107, optim_step_time=0.046, optim0_lr0=4.106e-04, train_time=1.061 +[ALab02] 2023-06-09 21:26:34,006 (trainer:721) INFO: 11epoch:train:4960-5510batch: iter_time=3.056e-04, forward_time=0.119, loss_ctc=3.105, loss=3.105, backward_time=0.109, optim_step_time=0.046, optim0_lr0=4.126e-04, train_time=1.090 +[ALab02] 2023-06-09 21:29:06,057 (trainer:721) INFO: 11epoch:train:5511-6061batch: iter_time=3.111e-04, forward_time=0.121, loss_ctc=3.149, loss=3.149, backward_time=0.109, optim_step_time=0.047, optim0_lr0=4.145e-04, train_time=1.103 +[ALab02] 2023-06-09 21:31:39,722 (trainer:721) INFO: 11epoch:train:6062-6612batch: iter_time=2.806e-04, forward_time=0.125, loss_ctc=3.182, loss=3.182, backward_time=0.106, optim_step_time=0.044, optim0_lr0=4.165e-04, train_time=1.116 +[ALab02] 2023-06-09 21:34:33,805 (trainer:721) INFO: 11epoch:train:6613-7163batch: iter_time=2.549e-04, forward_time=0.153, loss_ctc=2.959, loss=2.959, backward_time=0.109, optim_step_time=0.042, optim0_lr0=4.185e-04, train_time=1.264 +[ALab02] 2023-06-09 21:36:55,801 (trainer:721) INFO: 11epoch:train:7164-7714batch: iter_time=3.090e-04, forward_time=0.107, loss_ctc=3.237, loss=3.237, backward_time=0.108, optim_step_time=0.045, optim0_lr0=4.204e-04, train_time=1.032 +[ALab02] 2023-06-09 21:39:24,767 (trainer:721) INFO: 11epoch:train:7715-8265batch: iter_time=3.025e-04, forward_time=0.110, loss_ctc=3.012, loss=3.012, backward_time=0.118, optim_step_time=0.053, optim0_lr0=4.224e-04, train_time=1.081 +[ALab02] 2023-06-09 21:42:19,996 (trainer:721) INFO: 11epoch:train:8266-8816batch: iter_time=3.293e-04, forward_time=0.138, loss_ctc=3.050, loss=3.050, backward_time=0.133, optim_step_time=0.060, optim0_lr0=4.244e-04, train_time=1.271 +[ALab02] 2023-06-09 21:45:28,879 (trainer:721) INFO: 11epoch:train:8817-9367batch: iter_time=3.380e-04, forward_time=0.146, loss_ctc=3.043, loss=3.043, backward_time=0.148, optim_step_time=0.068, optim0_lr0=4.263e-04, train_time=1.369 +[ALab02] 2023-06-09 21:48:29,760 (trainer:721) INFO: 11epoch:train:9368-9918batch: iter_time=3.735e-04, forward_time=0.142, loss_ctc=2.994, loss=2.994, backward_time=0.143, optim_step_time=0.066, optim0_lr0=4.283e-04, train_time=1.315 +[ALab02] 2023-06-09 21:50:47,278 (trainer:721) INFO: 11epoch:train:9919-10469batch: iter_time=2.781e-04, forward_time=0.091, loss_ctc=3.026, loss=3.026, backward_time=0.126, optim_step_time=0.057, optim0_lr0=4.303e-04, train_time=0.999 +[ALab02] 2023-06-09 21:53:24,571 (trainer:721) INFO: 11epoch:train:10470-11020batch: iter_time=3.633e-04, forward_time=0.107, loss_ctc=3.047, loss=3.047, backward_time=0.141, optim_step_time=0.064, optim0_lr0=4.323e-04, train_time=1.141 +[ALab02] 2023-06-09 21:56:21,284 (trainer:338) INFO: 11epoch results: [train] iter_time=3.308e-04, forward_time=0.130, loss_ctc=3.093, loss=3.093, backward_time=0.116, optim_step_time=0.049, optim0_lr0=4.136e-04, train_time=1.168, time=53 minutes and 43.23 seconds, total_count=121319, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=5.469, cer_ctc=0.084, cer=0.084, loss=5.469, time=42.13 seconds, total_count=4851, gpu_max_cached_mem_GB=10.861, [att_plot] time=2 minutes and 11.04 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 21:56:24,902 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 21:56:24,903 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/1epoch.pth +[ALab02] 2023-06-09 21:56:24,903 (trainer:272) INFO: 12/70epoch started. Estimated time to finish: 2 days, 1 hour and 58 minutes +[ALab02] 2023-06-09 22:00:07,355 (trainer:721) INFO: 12epoch:train:1-551batch: iter_time=0.123, forward_time=0.118, loss_ctc=3.015, loss=3.015, backward_time=0.120, optim_step_time=0.053, optim0_lr0=4.342e-04, train_time=1.619 +[ALab02] 2023-06-09 22:02:41,844 (trainer:721) INFO: 12epoch:train:552-1102batch: iter_time=3.867e-04, forward_time=0.118, loss_ctc=2.874, loss=2.874, backward_time=0.120, optim_step_time=0.053, optim0_lr0=4.362e-04, train_time=1.121 +[ALab02] 2023-06-09 22:05:19,740 (trainer:721) INFO: 12epoch:train:1103-1653batch: iter_time=3.230e-04, forward_time=0.119, loss_ctc=2.914, loss=2.914, backward_time=0.124, optim_step_time=0.054, optim0_lr0=4.382e-04, train_time=1.145 +[ALab02] 2023-06-09 22:07:49,253 (trainer:721) INFO: 12epoch:train:1654-2204batch: iter_time=3.195e-04, forward_time=0.109, loss_ctc=2.977, loss=2.977, backward_time=0.121, optim_step_time=0.055, optim0_lr0=4.401e-04, train_time=1.084 +[ALab02] 2023-06-09 22:10:23,785 (trainer:721) INFO: 12epoch:train:2205-2755batch: iter_time=3.228e-04, forward_time=0.115, loss_ctc=3.030, loss=3.030, backward_time=0.122, optim_step_time=0.055, optim0_lr0=4.421e-04, train_time=1.123 +[ALab02] 2023-06-09 22:13:10,765 (trainer:721) INFO: 12epoch:train:2756-3306batch: iter_time=4.050e-04, forward_time=0.131, loss_ctc=3.030, loss=3.030, backward_time=0.125, optim_step_time=0.058, optim0_lr0=4.441e-04, train_time=1.212 +[ALab02] 2023-06-09 22:15:54,394 (trainer:721) INFO: 12epoch:train:3307-3857batch: iter_time=3.594e-04, forward_time=0.127, loss_ctc=3.061, loss=3.061, backward_time=0.125, optim_step_time=0.056, optim0_lr0=4.461e-04, train_time=1.186 +[ALab02] 2023-06-09 22:18:52,123 (trainer:721) INFO: 12epoch:train:3858-4408batch: iter_time=3.196e-04, forward_time=0.151, loss_ctc=3.088, loss=3.088, backward_time=0.120, optim_step_time=0.053, optim0_lr0=4.480e-04, train_time=1.290 +[ALab02] 2023-06-09 22:21:49,211 (trainer:721) INFO: 12epoch:train:4409-4959batch: iter_time=3.231e-04, forward_time=0.151, loss_ctc=2.981, loss=2.981, backward_time=0.119, optim_step_time=0.051, optim0_lr0=4.500e-04, train_time=1.283 +[ALab02] 2023-06-09 22:24:50,605 (trainer:721) INFO: 12epoch:train:4960-5510batch: iter_time=3.248e-04, forward_time=0.156, loss_ctc=2.964, loss=2.964, backward_time=0.119, optim_step_time=0.053, optim0_lr0=4.520e-04, train_time=1.318 +[ALab02] 2023-06-09 22:27:48,939 (trainer:721) INFO: 12epoch:train:5511-6061batch: iter_time=2.926e-04, forward_time=0.153, loss_ctc=2.929, loss=2.929, backward_time=0.118, optim_step_time=0.048, optim0_lr0=4.539e-04, train_time=1.295 +[ALab02] 2023-06-09 22:30:40,787 (trainer:721) INFO: 12epoch:train:6062-6612batch: iter_time=3.048e-04, forward_time=0.144, loss_ctc=2.959, loss=2.959, backward_time=0.118, optim_step_time=0.051, optim0_lr0=4.559e-04, train_time=1.246 +[ALab02] 2023-06-09 22:33:35,266 (trainer:721) INFO: 12epoch:train:6613-7163batch: iter_time=2.925e-04, forward_time=0.149, loss_ctc=2.978, loss=2.978, backward_time=0.117, optim_step_time=0.048, optim0_lr0=4.579e-04, train_time=1.266 +[ALab02] 2023-06-09 22:36:31,221 (trainer:721) INFO: 12epoch:train:7164-7714batch: iter_time=3.366e-04, forward_time=0.147, loss_ctc=2.985, loss=2.985, backward_time=0.121, optim_step_time=0.052, optim0_lr0=4.598e-04, train_time=1.278 +[ALab02] 2023-06-09 22:39:24,357 (trainer:721) INFO: 12epoch:train:7715-8265batch: iter_time=3.293e-04, forward_time=0.147, loss_ctc=2.963, loss=2.963, backward_time=0.117, optim_step_time=0.050, optim0_lr0=4.618e-04, train_time=1.256 +[ALab02] 2023-06-09 22:42:15,901 (trainer:721) INFO: 12epoch:train:8266-8816batch: iter_time=2.984e-04, forward_time=0.146, loss_ctc=3.008, loss=3.008, backward_time=0.114, optim_step_time=0.046, optim0_lr0=4.638e-04, train_time=1.246 +[ALab02] 2023-06-09 22:44:44,742 (trainer:721) INFO: 12epoch:train:8817-9367batch: iter_time=3.199e-04, forward_time=0.114, loss_ctc=2.974, loss=2.974, backward_time=0.114, optim_step_time=0.049, optim0_lr0=4.657e-04, train_time=1.081 +[ALab02] 2023-06-09 22:46:59,159 (trainer:721) INFO: 12epoch:train:9368-9918batch: iter_time=2.932e-04, forward_time=0.099, loss_ctc=3.020, loss=3.020, backward_time=0.107, optim_step_time=0.049, optim0_lr0=4.677e-04, train_time=0.976 +[ALab02] 2023-06-09 22:49:41,945 (trainer:721) INFO: 12epoch:train:9919-10469batch: iter_time=2.965e-04, forward_time=0.136, loss_ctc=2.910, loss=2.910, backward_time=0.110, optim_step_time=0.046, optim0_lr0=4.697e-04, train_time=1.181 +[ALab02] 2023-06-09 22:52:30,089 (trainer:721) INFO: 12epoch:train:10470-11020batch: iter_time=3.191e-04, forward_time=0.145, loss_ctc=3.011, loss=3.011, backward_time=0.108, optim_step_time=0.044, optim0_lr0=4.716e-04, train_time=1.219 +[ALab02] 2023-06-09 22:54:38,518 (trainer:338) INFO: 12epoch results: [train] iter_time=0.006, forward_time=0.134, loss_ctc=2.983, loss=2.983, backward_time=0.118, optim_step_time=0.051, optim0_lr0=4.530e-04, train_time=1.221, time=56 minutes and 8.4 seconds, total_count=132348, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=5.115, cer_ctc=0.082, cer=0.082, loss=5.115, time=55.54 seconds, total_count=5292, gpu_max_cached_mem_GB=10.861, [att_plot] time=1 minute and 9.67 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 22:54:41,830 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 22:54:41,831 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/2epoch.pth +[ALab02] 2023-06-09 22:54:41,832 (trainer:272) INFO: 13/70epoch started. Estimated time to finish: 2 days, 1 hour and 43 minutes +[ALab02] 2023-06-09 22:57:39,418 (trainer:721) INFO: 13epoch:train:1-551batch: iter_time=0.002, forward_time=0.159, loss_ctc=2.823, loss=2.823, backward_time=0.107, optim_step_time=0.041, optim0_lr0=4.736e-04, train_time=1.289 +[ALab02] 2023-06-09 23:00:30,136 (trainer:721) INFO: 13epoch:train:552-1102batch: iter_time=2.156e-04, forward_time=0.153, loss_ctc=2.838, loss=2.838, backward_time=0.103, optim_step_time=0.036, optim0_lr0=4.756e-04, train_time=1.237 +[ALab02] 2023-06-09 23:03:24,995 (trainer:721) INFO: 13epoch:train:1103-1653batch: iter_time=2.520e-04, forward_time=0.155, loss_ctc=2.777, loss=2.777, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.776e-04, train_time=1.271 +[ALab02] 2023-06-09 23:06:21,530 (trainer:721) INFO: 13epoch:train:1654-2204batch: iter_time=2.807e-04, forward_time=0.157, loss_ctc=2.724, loss=2.724, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.795e-04, train_time=1.281 +[ALab02] 2023-06-09 23:09:15,299 (trainer:721) INFO: 13epoch:train:2205-2755batch: iter_time=2.410e-04, forward_time=0.154, loss_ctc=2.795, loss=2.795, backward_time=0.106, optim_step_time=0.037, optim0_lr0=4.815e-04, train_time=1.263 +[ALab02] 2023-06-09 23:12:09,162 (trainer:721) INFO: 13epoch:train:2756-3306batch: iter_time=2.511e-04, forward_time=0.155, loss_ctc=2.704, loss=2.704, backward_time=0.107, optim_step_time=0.038, optim0_lr0=4.835e-04, train_time=1.260 +[ALab02] 2023-06-09 23:15:05,194 (trainer:721) INFO: 13epoch:train:3307-3857batch: iter_time=2.414e-04, forward_time=0.157, loss_ctc=2.762, loss=2.762, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.854e-04, train_time=1.279 +[ALab02] 2023-06-09 23:17:59,504 (trainer:721) INFO: 13epoch:train:3858-4408batch: iter_time=2.447e-04, forward_time=0.156, loss_ctc=2.789, loss=2.789, backward_time=0.105, optim_step_time=0.039, optim0_lr0=4.874e-04, train_time=1.264 +[ALab02] 2023-06-09 23:20:51,196 (trainer:721) INFO: 13epoch:train:4409-4959batch: iter_time=2.361e-04, forward_time=0.151, loss_ctc=2.756, loss=2.756, backward_time=0.107, optim_step_time=0.037, optim0_lr0=4.894e-04, train_time=1.244 +[ALab02] 2023-06-09 23:23:16,083 (trainer:721) INFO: 13epoch:train:4960-5510batch: iter_time=2.382e-04, forward_time=0.120, loss_ctc=2.741, loss=2.741, backward_time=0.099, optim_step_time=0.039, optim0_lr0=4.913e-04, train_time=1.054 +[ALab02] 2023-06-09 23:26:12,350 (trainer:721) INFO: 13epoch:train:5511-6061batch: iter_time=2.382e-04, forward_time=0.157, loss_ctc=2.763, loss=2.763, backward_time=0.109, optim_step_time=0.038, optim0_lr0=4.933e-04, train_time=1.280 +[ALab02] 2023-06-09 23:29:07,735 (trainer:721) INFO: 13epoch:train:6062-6612batch: iter_time=2.402e-04, forward_time=0.156, loss_ctc=2.761, loss=2.761, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.953e-04, train_time=1.272 +[ALab02] 2023-06-09 23:32:01,817 (trainer:721) INFO: 13epoch:train:6613-7163batch: iter_time=2.177e-04, forward_time=0.156, loss_ctc=2.805, loss=2.805, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.972e-04, train_time=1.267 +[ALab02] 2023-06-09 23:35:02,325 (trainer:721) INFO: 13epoch:train:7164-7714batch: iter_time=2.727e-04, forward_time=0.162, loss_ctc=2.725, loss=2.725, backward_time=0.110, optim_step_time=0.042, optim0_lr0=4.992e-04, train_time=1.306 +[ALab02] 2023-06-09 23:37:55,918 (trainer:721) INFO: 13epoch:train:7715-8265batch: iter_time=2.414e-04, forward_time=0.154, loss_ctc=2.658, loss=2.658, backward_time=0.107, optim_step_time=0.037, optim0_lr0=4.994e-04, train_time=1.261 +[ALab02] 2023-06-09 23:40:53,225 (trainer:721) INFO: 13epoch:train:8266-8816batch: iter_time=2.559e-04, forward_time=0.160, loss_ctc=2.709, loss=2.709, backward_time=0.106, optim_step_time=0.039, optim0_lr0=4.984e-04, train_time=1.286 +[ALab02] 2023-06-09 23:43:51,054 (trainer:721) INFO: 13epoch:train:8817-9367batch: iter_time=2.506e-04, forward_time=0.159, loss_ctc=2.692, loss=2.692, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.975e-04, train_time=1.290 +[ALab02] 2023-06-09 23:46:47,741 (trainer:721) INFO: 13epoch:train:9368-9918batch: iter_time=2.757e-04, forward_time=0.159, loss_ctc=2.623, loss=2.623, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.965e-04, train_time=1.284 +[ALab02] 2023-06-09 23:49:44,157 (trainer:721) INFO: 13epoch:train:9919-10469batch: iter_time=2.480e-04, forward_time=0.158, loss_ctc=2.666, loss=2.666, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.955e-04, train_time=1.281 +[ALab02] 2023-06-09 23:52:38,557 (trainer:721) INFO: 13epoch:train:10470-11020batch: iter_time=2.320e-04, forward_time=0.156, loss_ctc=2.607, loss=2.607, backward_time=0.106, optim_step_time=0.037, optim0_lr0=4.946e-04, train_time=1.265 +[ALab02] 2023-06-09 23:54:14,929 (trainer:338) INFO: 13epoch results: [train] iter_time=3.105e-04, forward_time=0.155, loss_ctc=2.735, loss=2.735, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.896e-04, train_time=1.262, time=57 minutes and 59.97 seconds, total_count=143377, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.920, cer_ctc=0.078, cer=0.078, loss=4.920, time=57.59 seconds, total_count=5733, gpu_max_cached_mem_GB=10.861, [att_plot] time=35.54 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-09 23:54:17,707 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-09 23:54:17,708 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/3epoch.pth +[ALab02] 2023-06-09 23:54:17,708 (trainer:272) INFO: 14/70epoch started. Estimated time to finish: 2 days, 1 hour and 28 minutes +[ALab02] 2023-06-09 23:56:42,663 (trainer:721) INFO: 14epoch:train:1-551batch: iter_time=0.001, forward_time=0.120, loss_ctc=2.579, loss=2.579, backward_time=0.099, optim_step_time=0.038, optim0_lr0=4.936e-04, train_time=1.051 +[ALab02] 2023-06-09 23:59:37,553 (trainer:721) INFO: 14epoch:train:552-1102batch: iter_time=2.545e-04, forward_time=0.157, loss_ctc=2.580, loss=2.580, backward_time=0.106, optim_step_time=0.040, optim0_lr0=4.927e-04, train_time=1.268 +[ALab02] 2023-06-10 00:02:33,311 (trainer:721) INFO: 14epoch:train:1103-1653batch: iter_time=2.523e-04, forward_time=0.157, loss_ctc=2.583, loss=2.583, backward_time=0.107, optim_step_time=0.040, optim0_lr0=4.917e-04, train_time=1.277 +[ALab02] 2023-06-10 00:05:30,202 (trainer:721) INFO: 14epoch:train:1654-2204batch: iter_time=2.454e-04, forward_time=0.159, loss_ctc=2.545, loss=2.545, backward_time=0.107, optim_step_time=0.041, optim0_lr0=4.908e-04, train_time=1.283 +[ALab02] 2023-06-10 00:08:24,953 (trainer:721) INFO: 14epoch:train:2205-2755batch: iter_time=2.422e-04, forward_time=0.156, loss_ctc=2.487, loss=2.487, backward_time=0.106, optim_step_time=0.038, optim0_lr0=4.899e-04, train_time=1.268 +[ALab02] 2023-06-10 00:11:22,847 (trainer:721) INFO: 14epoch:train:2756-3306batch: iter_time=2.582e-04, forward_time=0.160, loss_ctc=2.512, loss=2.512, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.890e-04, train_time=1.291 +[ALab02] 2023-06-10 00:14:17,026 (trainer:721) INFO: 14epoch:train:3307-3857batch: iter_time=2.414e-04, forward_time=0.155, loss_ctc=2.500, loss=2.500, backward_time=0.107, optim_step_time=0.038, optim0_lr0=4.880e-04, train_time=1.263 +[ALab02] 2023-06-10 00:17:17,590 (trainer:721) INFO: 14epoch:train:3858-4408batch: iter_time=2.618e-04, forward_time=0.162, loss_ctc=2.541, loss=2.541, backward_time=0.109, optim_step_time=0.042, optim0_lr0=4.871e-04, train_time=1.311 +[ALab02] 2023-06-10 00:20:17,791 (trainer:721) INFO: 14epoch:train:4409-4959batch: iter_time=2.782e-04, forward_time=0.162, loss_ctc=2.453, loss=2.453, backward_time=0.110, optim_step_time=0.040, optim0_lr0=4.862e-04, train_time=1.310 +[ALab02] 2023-06-10 00:23:15,165 (trainer:721) INFO: 14epoch:train:4960-5510batch: iter_time=2.431e-04, forward_time=0.159, loss_ctc=2.420, loss=2.420, backward_time=0.108, optim_step_time=0.041, optim0_lr0=4.853e-04, train_time=1.286 +[ALab02] 2023-06-10 00:26:12,573 (trainer:721) INFO: 14epoch:train:5511-6061batch: iter_time=2.725e-04, forward_time=0.158, loss_ctc=2.426, loss=2.426, backward_time=0.110, optim_step_time=0.040, optim0_lr0=4.844e-04, train_time=1.288 +[ALab02] 2023-06-10 00:29:10,182 (trainer:721) INFO: 14epoch:train:6062-6612batch: iter_time=2.593e-04, forward_time=0.159, loss_ctc=2.454, loss=2.454, backward_time=0.108, optim_step_time=0.041, optim0_lr0=4.835e-04, train_time=1.288 +[ALab02] 2023-06-10 00:31:38,723 (trainer:721) INFO: 14epoch:train:6613-7163batch: iter_time=2.531e-04, forward_time=0.123, loss_ctc=2.405, loss=2.405, backward_time=0.101, optim_step_time=0.040, optim0_lr0=4.826e-04, train_time=1.081 +[ALab02] 2023-06-10 00:34:31,655 (trainer:721) INFO: 14epoch:train:7164-7714batch: iter_time=2.478e-04, forward_time=0.152, loss_ctc=2.463, loss=2.463, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.818e-04, train_time=1.252 +[ALab02] 2023-06-10 00:37:32,060 (trainer:721) INFO: 14epoch:train:7715-8265batch: iter_time=2.564e-04, forward_time=0.161, loss_ctc=2.516, loss=2.516, backward_time=0.111, optim_step_time=0.041, optim0_lr0=4.809e-04, train_time=1.308 +[ALab02] 2023-06-10 00:40:29,703 (trainer:721) INFO: 14epoch:train:8266-8816batch: iter_time=2.475e-04, forward_time=0.158, loss_ctc=2.463, loss=2.463, backward_time=0.110, optim_step_time=0.041, optim0_lr0=4.800e-04, train_time=1.290 +[ALab02] 2023-06-10 00:43:28,904 (trainer:721) INFO: 14epoch:train:8817-9367batch: iter_time=2.647e-04, forward_time=0.161, loss_ctc=2.487, loss=2.487, backward_time=0.110, optim_step_time=0.041, optim0_lr0=4.791e-04, train_time=1.301 +[ALab02] 2023-06-10 00:46:30,323 (trainer:721) INFO: 14epoch:train:9368-9918batch: iter_time=2.630e-04, forward_time=0.162, loss_ctc=2.459, loss=2.459, backward_time=0.111, optim_step_time=0.041, optim0_lr0=4.783e-04, train_time=1.317 +[ALab02] 2023-06-10 00:49:28,294 (trainer:721) INFO: 14epoch:train:9919-10469batch: iter_time=2.493e-04, forward_time=0.160, loss_ctc=2.440, loss=2.440, backward_time=0.109, optim_step_time=0.039, optim0_lr0=4.774e-04, train_time=1.291 +[ALab02] 2023-06-10 00:52:25,142 (trainer:721) INFO: 14epoch:train:10470-11020batch: iter_time=2.598e-04, forward_time=0.159, loss_ctc=2.439, loss=2.439, backward_time=0.107, optim_step_time=0.040, optim0_lr0=4.766e-04, train_time=1.284 +[ALab02] 2023-06-10 00:53:59,361 (trainer:338) INFO: 14epoch results: [train] iter_time=2.985e-04, forward_time=0.155, loss_ctc=2.488, loss=2.488, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.849e-04, train_time=1.266, time=58 minutes and 10.73 seconds, total_count=154406, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.514, cer_ctc=0.072, cer=0.072, loss=4.514, time=57.84 seconds, total_count=6174, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.08 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 00:54:02,085 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 00:54:02,087 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/4epoch.pth +[ALab02] 2023-06-10 00:54:02,087 (trainer:272) INFO: 15/70epoch started. Estimated time to finish: 2 days, 1 hour and 6 minutes +[ALab02] 2023-06-10 00:57:00,308 (trainer:721) INFO: 15epoch:train:1-551batch: iter_time=0.001, forward_time=0.158, loss_ctc=2.365, loss=2.365, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.757e-04, train_time=1.295 +[ALab02] 2023-06-10 01:00:10,191 (trainer:721) INFO: 15epoch:train:552-1102batch: iter_time=2.968e-04, forward_time=0.167, loss_ctc=2.348, loss=2.348, backward_time=0.122, optim_step_time=0.047, optim0_lr0=4.749e-04, train_time=1.376 +[ALab02] 2023-06-10 01:03:06,223 (trainer:721) INFO: 15epoch:train:1103-1653batch: iter_time=2.545e-04, forward_time=0.158, loss_ctc=2.405, loss=2.405, backward_time=0.107, optim_step_time=0.040, optim0_lr0=4.740e-04, train_time=1.278 +[ALab02] 2023-06-10 01:05:54,658 (trainer:721) INFO: 15epoch:train:1654-2204batch: iter_time=2.712e-04, forward_time=0.147, loss_ctc=2.419, loss=2.419, backward_time=0.108, optim_step_time=0.042, optim0_lr0=4.732e-04, train_time=1.223 +[ALab02] 2023-06-10 01:08:35,634 (trainer:721) INFO: 15epoch:train:2205-2755batch: iter_time=2.609e-04, forward_time=0.139, loss_ctc=2.336, loss=2.336, backward_time=0.104, optim_step_time=0.042, optim0_lr0=4.723e-04, train_time=1.167 +[ALab02] 2023-06-10 01:11:35,966 (trainer:721) INFO: 15epoch:train:2756-3306batch: iter_time=2.518e-04, forward_time=0.162, loss_ctc=2.308, loss=2.308, backward_time=0.111, optim_step_time=0.041, optim0_lr0=4.715e-04, train_time=1.310 +[ALab02] 2023-06-10 01:14:34,152 (trainer:721) INFO: 15epoch:train:3307-3857batch: iter_time=2.497e-04, forward_time=0.160, loss_ctc=2.278, loss=2.278, backward_time=0.109, optim_step_time=0.042, optim0_lr0=4.707e-04, train_time=1.294 +[ALab02] 2023-06-10 01:17:32,585 (trainer:721) INFO: 15epoch:train:3858-4408batch: iter_time=2.586e-04, forward_time=0.159, loss_ctc=2.395, loss=2.395, backward_time=0.110, optim_step_time=0.043, optim0_lr0=4.699e-04, train_time=1.294 +[ALab02] 2023-06-10 01:20:32,405 (trainer:721) INFO: 15epoch:train:4409-4959batch: iter_time=2.853e-04, forward_time=0.160, loss_ctc=2.270, loss=2.270, backward_time=0.111, optim_step_time=0.042, optim0_lr0=4.691e-04, train_time=1.306 +[ALab02] 2023-06-10 01:23:31,122 (trainer:721) INFO: 15epoch:train:4960-5510batch: iter_time=2.877e-04, forward_time=0.161, loss_ctc=2.274, loss=2.274, backward_time=0.109, optim_step_time=0.041, optim0_lr0=4.682e-04, train_time=1.295 +[ALab02] 2023-06-10 01:26:27,528 (trainer:721) INFO: 15epoch:train:5511-6061batch: iter_time=2.468e-04, forward_time=0.157, loss_ctc=2.336, loss=2.336, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.674e-04, train_time=1.281 +[ALab02] 2023-06-10 01:29:25,993 (trainer:721) INFO: 15epoch:train:6062-6612batch: iter_time=2.584e-04, forward_time=0.159, loss_ctc=2.287, loss=2.287, backward_time=0.109, optim_step_time=0.041, optim0_lr0=4.666e-04, train_time=1.296 +[ALab02] 2023-06-10 01:32:23,054 (trainer:721) INFO: 15epoch:train:6613-7163batch: iter_time=2.550e-04, forward_time=0.158, loss_ctc=2.277, loss=2.277, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.658e-04, train_time=1.284 +[ALab02] 2023-06-10 01:35:20,071 (trainer:721) INFO: 15epoch:train:7164-7714batch: iter_time=2.555e-04, forward_time=0.159, loss_ctc=2.258, loss=2.258, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.650e-04, train_time=1.285 +[ALab02] 2023-06-10 01:38:16,396 (trainer:721) INFO: 15epoch:train:7715-8265batch: iter_time=2.358e-04, forward_time=0.159, loss_ctc=2.247, loss=2.247, backward_time=0.107, optim_step_time=0.038, optim0_lr0=4.643e-04, train_time=1.280 +[ALab02] 2023-06-10 01:41:01,482 (trainer:721) INFO: 15epoch:train:8266-8816batch: iter_time=2.634e-04, forward_time=0.143, loss_ctc=2.184, loss=2.184, backward_time=0.106, optim_step_time=0.042, optim0_lr0=4.635e-04, train_time=1.198 +[ALab02] 2023-06-10 01:43:47,143 (trainer:721) INFO: 15epoch:train:8817-9367batch: iter_time=2.979e-04, forward_time=0.144, loss_ctc=2.221, loss=2.221, backward_time=0.107, optim_step_time=0.044, optim0_lr0=4.627e-04, train_time=1.203 +[ALab02] 2023-06-10 01:46:44,436 (trainer:721) INFO: 15epoch:train:9368-9918batch: iter_time=2.565e-04, forward_time=0.158, loss_ctc=2.200, loss=2.200, backward_time=0.108, optim_step_time=0.041, optim0_lr0=4.619e-04, train_time=1.286 +[ALab02] 2023-06-10 01:49:44,332 (trainer:721) INFO: 15epoch:train:9919-10469batch: iter_time=2.637e-04, forward_time=0.161, loss_ctc=2.318, loss=2.318, backward_time=0.109, optim_step_time=0.041, optim0_lr0=4.611e-04, train_time=1.306 +[ALab02] 2023-06-10 01:52:42,791 (trainer:721) INFO: 15epoch:train:10470-11020batch: iter_time=2.674e-04, forward_time=0.160, loss_ctc=2.190, loss=2.190, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.604e-04, train_time=1.294 +[ALab02] 2023-06-10 01:54:19,600 (trainer:338) INFO: 15epoch results: [train] iter_time=3.111e-04, forward_time=0.156, loss_ctc=2.295, loss=2.295, backward_time=0.109, optim_step_time=0.041, optim0_lr0=4.679e-04, train_time=1.278, time=58 minutes and 44.07 seconds, total_count=165435, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.544, cer_ctc=0.070, cer=0.070, loss=4.544, time=56.61 seconds, total_count=6615, gpu_max_cached_mem_GB=10.861, [att_plot] time=36.83 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 01:54:22,305 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 01:54:22,308 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/5epoch.pth +[ALab02] 2023-06-10 01:54:22,308 (trainer:272) INFO: 16/70epoch started. Estimated time to finish: 2 days, 42 minutes and 20.29 seconds +[ALab02] 2023-06-10 01:57:20,870 (trainer:721) INFO: 16epoch:train:1-551batch: iter_time=0.001, forward_time=0.160, loss_ctc=2.232, loss=2.232, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.596e-04, train_time=1.297 +[ALab02] 2023-06-10 02:00:21,239 (trainer:721) INFO: 16epoch:train:552-1102batch: iter_time=2.821e-04, forward_time=0.164, loss_ctc=2.150, loss=2.150, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.588e-04, train_time=1.308 +[ALab02] 2023-06-10 02:03:16,194 (trainer:721) INFO: 16epoch:train:1103-1653batch: iter_time=2.515e-04, forward_time=0.156, loss_ctc=2.113, loss=2.113, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.581e-04, train_time=1.270 +[ALab02] 2023-06-10 02:06:08,211 (trainer:721) INFO: 16epoch:train:1654-2204batch: iter_time=2.513e-04, forward_time=0.155, loss_ctc=2.099, loss=2.099, backward_time=0.104, optim_step_time=0.038, optim0_lr0=4.573e-04, train_time=1.249 +[ALab02] 2023-06-10 02:09:00,631 (trainer:721) INFO: 16epoch:train:2205-2755batch: iter_time=2.253e-04, forward_time=0.155, loss_ctc=2.195, loss=2.195, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.566e-04, train_time=1.251 +[ALab02] 2023-06-10 02:12:00,417 (trainer:721) INFO: 16epoch:train:2756-3306batch: iter_time=2.540e-04, forward_time=0.162, loss_ctc=2.131, loss=2.131, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.558e-04, train_time=1.306 +[ALab02] 2023-06-10 02:14:57,487 (trainer:721) INFO: 16epoch:train:3307-3857batch: iter_time=2.573e-04, forward_time=0.160, loss_ctc=2.192, loss=2.192, backward_time=0.107, optim_step_time=0.040, optim0_lr0=4.551e-04, train_time=1.286 +[ALab02] 2023-06-10 02:17:25,207 (trainer:721) INFO: 16epoch:train:3858-4408batch: iter_time=2.666e-04, forward_time=0.122, loss_ctc=2.178, loss=2.178, backward_time=0.101, optim_step_time=0.041, optim0_lr0=4.543e-04, train_time=1.071 +[ALab02] 2023-06-10 02:20:23,300 (trainer:721) INFO: 16epoch:train:4409-4959batch: iter_time=2.420e-04, forward_time=0.160, loss_ctc=2.157, loss=2.157, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.536e-04, train_time=1.294 +[ALab02] 2023-06-10 02:23:18,742 (trainer:721) INFO: 16epoch:train:4960-5510batch: iter_time=2.637e-04, forward_time=0.155, loss_ctc=2.194, loss=2.194, backward_time=0.109, optim_step_time=0.039, optim0_lr0=4.529e-04, train_time=1.272 +[ALab02] 2023-06-10 02:26:16,456 (trainer:721) INFO: 16epoch:train:5511-6061batch: iter_time=2.747e-04, forward_time=0.159, loss_ctc=2.054, loss=2.054, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.521e-04, train_time=1.290 +[ALab02] 2023-06-10 02:29:12,915 (trainer:721) INFO: 16epoch:train:6062-6612batch: iter_time=2.658e-04, forward_time=0.156, loss_ctc=2.079, loss=2.079, backward_time=0.109, optim_step_time=0.039, optim0_lr0=4.514e-04, train_time=1.280 +[ALab02] 2023-06-10 02:32:10,220 (trainer:721) INFO: 16epoch:train:6613-7163batch: iter_time=2.619e-04, forward_time=0.158, loss_ctc=2.139, loss=2.139, backward_time=0.108, optim_step_time=0.042, optim0_lr0=4.507e-04, train_time=1.287 +[ALab02] 2023-06-10 02:35:08,529 (trainer:721) INFO: 16epoch:train:7164-7714batch: iter_time=2.554e-04, forward_time=0.161, loss_ctc=2.091, loss=2.091, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.500e-04, train_time=1.294 +[ALab02] 2023-06-10 02:38:05,157 (trainer:721) INFO: 16epoch:train:7715-8265batch: iter_time=2.574e-04, forward_time=0.156, loss_ctc=2.150, loss=2.150, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.493e-04, train_time=1.282 +[ALab02] 2023-06-10 02:41:06,788 (trainer:721) INFO: 16epoch:train:8266-8816batch: iter_time=2.605e-04, forward_time=0.163, loss_ctc=2.105, loss=2.105, backward_time=0.111, optim_step_time=0.042, optim0_lr0=4.485e-04, train_time=1.318 +[ALab02] 2023-06-10 02:44:06,780 (trainer:721) INFO: 16epoch:train:8817-9367batch: iter_time=2.524e-04, forward_time=0.162, loss_ctc=2.068, loss=2.068, backward_time=0.109, optim_step_time=0.041, optim0_lr0=4.478e-04, train_time=1.307 +[ALab02] 2023-06-10 02:47:02,952 (trainer:721) INFO: 16epoch:train:9368-9918batch: iter_time=2.685e-04, forward_time=0.157, loss_ctc=2.057, loss=2.057, backward_time=0.108, optim_step_time=0.039, optim0_lr0=4.471e-04, train_time=1.278 +[ALab02] 2023-06-10 02:50:00,620 (trainer:721) INFO: 16epoch:train:9919-10469batch: iter_time=2.518e-04, forward_time=0.158, loss_ctc=2.058, loss=2.058, backward_time=0.109, optim_step_time=0.040, optim0_lr0=4.464e-04, train_time=1.289 +[ALab02] 2023-06-10 02:52:25,908 (trainer:721) INFO: 16epoch:train:10470-11020batch: iter_time=2.520e-04, forward_time=0.120, loss_ctc=2.066, loss=2.066, backward_time=0.100, optim_step_time=0.038, optim0_lr0=4.457e-04, train_time=1.054 +[ALab02] 2023-06-10 02:53:58,419 (trainer:338) INFO: 16epoch results: [train] iter_time=3.036e-04, forward_time=0.155, loss_ctc=2.126, loss=2.126, backward_time=0.107, optim_step_time=0.040, optim0_lr0=4.526e-04, train_time=1.264, time=58 minutes and 6.92 seconds, total_count=176464, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.278, cer_ctc=0.067, cer=0.067, loss=4.278, time=55.64 seconds, total_count=7056, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.56 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 02:54:01,312 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 02:54:01,315 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/6epoch.pth +[ALab02] 2023-06-10 02:54:01,315 (trainer:272) INFO: 17/70epoch started. Estimated time to finish: 2 days, 11 minutes and 11.92 seconds +[ALab02] 2023-06-10 02:57:00,885 (trainer:721) INFO: 17epoch:train:1-551batch: iter_time=0.001, forward_time=0.160, loss_ctc=1.953, loss=1.953, backward_time=0.110, optim_step_time=0.040, optim0_lr0=4.450e-04, train_time=1.303 +[ALab02] 2023-06-10 02:59:57,699 (trainer:721) INFO: 17epoch:train:552-1102batch: iter_time=2.632e-04, forward_time=0.159, loss_ctc=1.892, loss=1.892, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.443e-04, train_time=1.283 +[ALab02] 2023-06-10 03:03:00,505 (trainer:721) INFO: 17epoch:train:1103-1653batch: iter_time=3.059e-04, forward_time=0.164, loss_ctc=1.987, loss=1.987, backward_time=0.113, optim_step_time=0.043, optim0_lr0=4.436e-04, train_time=1.327 +[ALab02] 2023-06-10 03:05:56,005 (trainer:721) INFO: 17epoch:train:1654-2204batch: iter_time=2.509e-04, forward_time=0.156, loss_ctc=1.954, loss=1.954, backward_time=0.108, optim_step_time=0.038, optim0_lr0=4.430e-04, train_time=1.273 +[ALab02] 2023-06-10 03:08:53,363 (trainer:721) INFO: 17epoch:train:2205-2755batch: iter_time=2.630e-04, forward_time=0.158, loss_ctc=2.010, loss=2.010, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.423e-04, train_time=1.289 +[ALab02] 2023-06-10 03:11:48,562 (trainer:721) INFO: 17epoch:train:2756-3306batch: iter_time=2.627e-04, forward_time=0.156, loss_ctc=2.060, loss=2.060, backward_time=0.107, optim_step_time=0.041, optim0_lr0=4.416e-04, train_time=1.272 +[ALab02] 2023-06-10 03:14:46,823 (trainer:721) INFO: 17epoch:train:3307-3857batch: iter_time=2.453e-04, forward_time=0.160, loss_ctc=2.051, loss=2.051, backward_time=0.109, optim_step_time=0.039, optim0_lr0=4.409e-04, train_time=1.292 +[ALab02] 2023-06-10 03:17:45,882 (trainer:721) INFO: 17epoch:train:3858-4408batch: iter_time=2.761e-04, forward_time=0.158, loss_ctc=2.068, loss=2.068, backward_time=0.111, optim_step_time=0.040, optim0_lr0=4.402e-04, train_time=1.299 +[ALab02] 2023-06-10 03:20:42,601 (trainer:721) INFO: 17epoch:train:4409-4959batch: iter_time=2.477e-04, forward_time=0.157, loss_ctc=1.994, loss=1.994, backward_time=0.108, optim_step_time=0.040, optim0_lr0=4.396e-04, train_time=1.284 +[ALab02] 2023-06-10 03:23:40,056 (trainer:721) INFO: 17epoch:train:4960-5510batch: iter_time=2.517e-04, forward_time=0.158, loss_ctc=2.094, loss=2.094, backward_time=0.109, optim_step_time=0.039, optim0_lr0=4.389e-04, train_time=1.287 +[ALab02] 2023-06-10 03:26:06,972 (trainer:721) INFO: 17epoch:train:5511-6061batch: iter_time=2.492e-04, forward_time=0.122, loss_ctc=2.010, loss=2.010, backward_time=0.101, optim_step_time=0.041, optim0_lr0=4.382e-04, train_time=1.067 +[ALab02] 2023-06-10 03:29:00,559 (trainer:721) INFO: 17epoch:train:6062-6612batch: iter_time=2.402e-04, forward_time=0.154, loss_ctc=1.976, loss=1.976, backward_time=0.106, optim_step_time=0.039, optim0_lr0=4.376e-04, train_time=1.259 +[ALab02] 2023-06-10 03:31:53,664 (trainer:721) INFO: 17epoch:train:6613-7163batch: iter_time=2.251e-04, forward_time=0.155, loss_ctc=1.920, loss=1.920, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.369e-04, train_time=1.258 +[ALab02] 2023-06-10 03:34:47,598 (trainer:721) INFO: 17epoch:train:7164-7714batch: iter_time=2.322e-04, forward_time=0.156, loss_ctc=1.883, loss=1.883, backward_time=0.104, optim_step_time=0.038, optim0_lr0=4.363e-04, train_time=1.263 +[ALab02] 2023-06-10 03:37:41,317 (trainer:721) INFO: 17epoch:train:7715-8265batch: iter_time=2.348e-04, forward_time=0.156, loss_ctc=1.889, loss=1.889, backward_time=0.105, optim_step_time=0.036, optim0_lr0=4.356e-04, train_time=1.258 +[ALab02] 2023-06-10 03:40:33,354 (trainer:721) INFO: 17epoch:train:8266-8816batch: iter_time=2.417e-04, forward_time=0.153, loss_ctc=1.911, loss=1.911, backward_time=0.105, optim_step_time=0.036, optim0_lr0=4.350e-04, train_time=1.249 +[ALab02] 2023-06-10 03:43:23,501 (trainer:721) INFO: 17epoch:train:8817-9367batch: iter_time=2.205e-04, forward_time=0.152, loss_ctc=1.895, loss=1.895, backward_time=0.103, optim_step_time=0.036, optim0_lr0=4.343e-04, train_time=1.234 +[ALab02] 2023-06-10 03:46:16,598 (trainer:721) INFO: 17epoch:train:9368-9918batch: iter_time=2.300e-04, forward_time=0.154, loss_ctc=1.945, loss=1.945, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.337e-04, train_time=1.255 +[ALab02] 2023-06-10 03:49:10,701 (trainer:721) INFO: 17epoch:train:9919-10469batch: iter_time=2.540e-04, forward_time=0.157, loss_ctc=1.896, loss=1.896, backward_time=0.104, optim_step_time=0.036, optim0_lr0=4.330e-04, train_time=1.265 +[ALab02] 2023-06-10 03:52:03,923 (trainer:721) INFO: 17epoch:train:10470-11020batch: iter_time=2.258e-04, forward_time=0.156, loss_ctc=1.941, loss=1.941, backward_time=0.104, optim_step_time=0.038, optim0_lr0=4.324e-04, train_time=1.257 +[ALab02] 2023-06-10 03:53:35,921 (trainer:338) INFO: 17epoch results: [train] iter_time=2.934e-04, forward_time=0.155, loss_ctc=1.966, loss=1.966, backward_time=0.107, optim_step_time=0.039, optim0_lr0=4.386e-04, train_time=1.264, time=58 minutes and 6.06 seconds, total_count=187493, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.152, cer_ctc=0.065, cer=0.065, loss=4.152, time=56.32 seconds, total_count=7497, gpu_max_cached_mem_GB=10.861, [att_plot] time=32.23 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 03:53:38,325 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 03:53:38,326 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/7epoch.pth +[ALab02] 2023-06-10 03:53:38,326 (trainer:272) INFO: 18/70epoch started. Estimated time to finish: 1 day, 23 hours and 36 minutes +[ALab02] 2023-06-10 03:56:32,128 (trainer:721) INFO: 18epoch:train:1-551batch: iter_time=0.001, forward_time=0.155, loss_ctc=1.802, loss=1.802, backward_time=0.104, optim_step_time=0.038, optim0_lr0=4.318e-04, train_time=1.262 +[ALab02] 2023-06-10 03:59:21,169 (trainer:721) INFO: 18epoch:train:552-1102batch: iter_time=4.528e-04, forward_time=0.150, loss_ctc=1.822, loss=1.822, backward_time=0.103, optim_step_time=0.037, optim0_lr0=4.311e-04, train_time=1.227 +[ALab02] 2023-06-10 04:01:45,924 (trainer:721) INFO: 18epoch:train:1103-1653batch: iter_time=2.409e-04, forward_time=0.120, loss_ctc=1.847, loss=1.847, backward_time=0.098, optim_step_time=0.037, optim0_lr0=4.305e-04, train_time=1.050 +[ALab02] 2023-06-10 04:04:38,646 (trainer:721) INFO: 18epoch:train:1654-2204batch: iter_time=2.140e-04, forward_time=0.154, loss_ctc=1.809, loss=1.809, backward_time=0.105, optim_step_time=0.036, optim0_lr0=4.299e-04, train_time=1.253 +[ALab02] 2023-06-10 04:07:32,964 (trainer:721) INFO: 18epoch:train:2205-2755batch: iter_time=2.292e-04, forward_time=0.157, loss_ctc=1.813, loss=1.813, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.292e-04, train_time=1.266 +[ALab02] 2023-06-10 04:10:28,502 (trainer:721) INFO: 18epoch:train:2756-3306batch: iter_time=2.406e-04, forward_time=0.159, loss_ctc=1.774, loss=1.774, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.286e-04, train_time=1.272 +[ALab02] 2023-06-10 04:13:19,412 (trainer:721) INFO: 18epoch:train:3307-3857batch: iter_time=2.234e-04, forward_time=0.152, loss_ctc=1.828, loss=1.828, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.280e-04, train_time=1.242 +[ALab02] 2023-06-10 04:16:14,205 (trainer:721) INFO: 18epoch:train:3858-4408batch: iter_time=2.333e-04, forward_time=0.155, loss_ctc=1.804, loss=1.804, backward_time=0.107, optim_step_time=0.038, optim0_lr0=4.274e-04, train_time=1.268 +[ALab02] 2023-06-10 04:19:09,121 (trainer:721) INFO: 18epoch:train:4409-4959batch: iter_time=2.353e-04, forward_time=0.157, loss_ctc=1.772, loss=1.772, backward_time=0.105, optim_step_time=0.036, optim0_lr0=4.268e-04, train_time=1.269 +[ALab02] 2023-06-10 04:22:03,097 (trainer:721) INFO: 18epoch:train:4960-5510batch: iter_time=2.462e-04, forward_time=0.155, loss_ctc=1.827, loss=1.827, backward_time=0.106, optim_step_time=0.036, optim0_lr0=4.262e-04, train_time=1.264 +[ALab02] 2023-06-10 04:24:53,118 (trainer:721) INFO: 18epoch:train:5511-6061batch: iter_time=2.142e-04, forward_time=0.153, loss_ctc=1.747, loss=1.747, backward_time=0.102, optim_step_time=0.036, optim0_lr0=4.256e-04, train_time=1.234 +[ALab02] 2023-06-10 04:27:45,849 (trainer:721) INFO: 18epoch:train:6062-6612batch: iter_time=2.366e-04, forward_time=0.156, loss_ctc=1.747, loss=1.747, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.249e-04, train_time=1.253 +[ALab02] 2023-06-10 04:30:38,414 (trainer:721) INFO: 18epoch:train:6613-7163batch: iter_time=2.320e-04, forward_time=0.154, loss_ctc=1.707, loss=1.707, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.243e-04, train_time=1.255 +[ALab02] 2023-06-10 04:33:30,753 (trainer:721) INFO: 18epoch:train:7164-7714batch: iter_time=2.322e-04, forward_time=0.153, loss_ctc=1.783, loss=1.783, backward_time=0.105, optim_step_time=0.038, optim0_lr0=4.237e-04, train_time=1.249 +[ALab02] 2023-06-10 04:35:42,727 (trainer:721) INFO: 18epoch:train:7715-8265batch: iter_time=2.219e-04, forward_time=0.106, loss_ctc=1.748, loss=1.748, backward_time=0.094, optim_step_time=0.037, optim0_lr0=4.231e-04, train_time=0.959 +[ALab02] 2023-06-10 04:38:33,649 (trainer:721) INFO: 18epoch:train:8266-8816batch: iter_time=2.365e-04, forward_time=0.152, loss_ctc=1.760, loss=1.760, backward_time=0.105, optim_step_time=0.036, optim0_lr0=4.226e-04, train_time=1.240 +[ALab02] 2023-06-10 04:41:27,341 (trainer:721) INFO: 18epoch:train:8817-9367batch: iter_time=2.419e-04, forward_time=0.157, loss_ctc=1.716, loss=1.716, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.220e-04, train_time=1.262 +[ALab02] 2023-06-10 04:44:21,996 (trainer:721) INFO: 18epoch:train:9368-9918batch: iter_time=2.487e-04, forward_time=0.156, loss_ctc=1.726, loss=1.726, backward_time=0.106, optim_step_time=0.036, optim0_lr0=4.214e-04, train_time=1.266 +[ALab02] 2023-06-10 04:47:15,119 (trainer:721) INFO: 18epoch:train:9919-10469batch: iter_time=2.263e-04, forward_time=0.155, loss_ctc=1.785, loss=1.785, backward_time=0.105, optim_step_time=0.037, optim0_lr0=4.208e-04, train_time=1.257 +[ALab02] 2023-06-10 04:50:11,275 (trainer:721) INFO: 18epoch:train:10470-11020batch: iter_time=2.239e-04, forward_time=0.157, loss_ctc=1.746, loss=1.746, backward_time=0.107, optim_step_time=0.037, optim0_lr0=4.202e-04, train_time=1.278 +[ALab02] 2023-06-10 04:51:39,062 (trainer:338) INFO: 18epoch results: [train] iter_time=2.925e-04, forward_time=0.151, loss_ctc=1.778, loss=1.778, backward_time=0.104, optim_step_time=0.037, optim0_lr0=4.259e-04, train_time=1.231, time=56 minutes and 36 seconds, total_count=198522, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.109, cer_ctc=0.063, cer=0.063, loss=4.109, time=55.27 seconds, total_count=7938, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.47 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 04:51:41,494 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 04:51:41,495 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/8epoch.pth +[ALab02] 2023-06-10 04:51:41,495 (trainer:272) INFO: 19/70epoch started. Estimated time to finish: 1 day, 22 hours and 54 minutes +[ALab02] 2023-06-10 04:54:27,884 (trainer:721) INFO: 19epoch:train:1-551batch: iter_time=6.842e-04, forward_time=0.149, loss_ctc=1.659, loss=1.659, backward_time=0.099, optim_step_time=0.032, optim0_lr0=4.196e-04, train_time=1.207 +[ALab02] 2023-06-10 04:57:13,023 (trainer:721) INFO: 19epoch:train:552-1102batch: iter_time=1.720e-04, forward_time=0.147, loss_ctc=1.660, loss=1.660, backward_time=0.100, optim_step_time=0.033, optim0_lr0=4.190e-04, train_time=1.201 +[ALab02] 2023-06-10 04:59:58,212 (trainer:721) INFO: 19epoch:train:1103-1653batch: iter_time=1.688e-04, forward_time=0.147, loss_ctc=1.702, loss=1.702, backward_time=0.098, optim_step_time=0.033, optim0_lr0=4.184e-04, train_time=1.195 +[ALab02] 2023-06-10 05:02:44,416 (trainer:721) INFO: 19epoch:train:1654-2204batch: iter_time=1.701e-04, forward_time=0.149, loss_ctc=1.619, loss=1.619, backward_time=0.099, optim_step_time=0.033, optim0_lr0=4.179e-04, train_time=1.208 +[ALab02] 2023-06-10 05:05:27,736 (trainer:721) INFO: 19epoch:train:2205-2755batch: iter_time=1.625e-04, forward_time=0.148, loss_ctc=1.592, loss=1.592, backward_time=0.097, optim_step_time=0.033, optim0_lr0=4.173e-04, train_time=1.185 +[ALab02] 2023-06-10 05:08:14,788 (trainer:721) INFO: 19epoch:train:2756-3306batch: iter_time=1.715e-04, forward_time=0.150, loss_ctc=1.711, loss=1.711, backward_time=0.099, optim_step_time=0.033, optim0_lr0=4.167e-04, train_time=1.212 +[ALab02] 2023-06-10 05:10:29,428 (trainer:721) INFO: 19epoch:train:3307-3857batch: iter_time=1.880e-04, forward_time=0.111, loss_ctc=1.653, loss=1.653, backward_time=0.091, optim_step_time=0.035, optim0_lr0=4.162e-04, train_time=0.978 +[ALab02] 2023-06-10 05:13:18,328 (trainer:721) INFO: 19epoch:train:3858-4408batch: iter_time=1.959e-04, forward_time=0.153, loss_ctc=1.622, loss=1.622, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.156e-04, train_time=1.226 +[ALab02] 2023-06-10 05:16:05,693 (trainer:721) INFO: 19epoch:train:4409-4959batch: iter_time=1.913e-04, forward_time=0.150, loss_ctc=1.703, loss=1.703, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.150e-04, train_time=1.215 +[ALab02] 2023-06-10 05:18:53,807 (trainer:721) INFO: 19epoch:train:4960-5510batch: iter_time=1.942e-04, forward_time=0.149, loss_ctc=1.614, loss=1.614, backward_time=0.101, optim_step_time=0.034, optim0_lr0=4.145e-04, train_time=1.222 +[ALab02] 2023-06-10 05:21:39,821 (trainer:721) INFO: 19epoch:train:5511-6061batch: iter_time=1.984e-04, forward_time=0.149, loss_ctc=1.680, loss=1.680, backward_time=0.099, optim_step_time=0.034, optim0_lr0=4.139e-04, train_time=1.204 +[ALab02] 2023-06-10 05:24:26,329 (trainer:721) INFO: 19epoch:train:6062-6612batch: iter_time=1.951e-04, forward_time=0.148, loss_ctc=1.665, loss=1.665, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.134e-04, train_time=1.208 +[ALab02] 2023-06-10 05:27:15,279 (trainer:721) INFO: 19epoch:train:6613-7163batch: iter_time=1.940e-04, forward_time=0.152, loss_ctc=1.625, loss=1.625, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.128e-04, train_time=1.226 +[ALab02] 2023-06-10 05:30:02,878 (trainer:721) INFO: 19epoch:train:7164-7714batch: iter_time=1.723e-04, forward_time=0.150, loss_ctc=1.651, loss=1.651, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.122e-04, train_time=1.217 +[ALab02] 2023-06-10 05:32:46,137 (trainer:721) INFO: 19epoch:train:7715-8265batch: iter_time=1.919e-04, forward_time=0.145, loss_ctc=1.702, loss=1.702, backward_time=0.098, optim_step_time=0.033, optim0_lr0=4.117e-04, train_time=1.185 +[ALab02] 2023-06-10 05:35:33,380 (trainer:721) INFO: 19epoch:train:8266-8816batch: iter_time=1.897e-04, forward_time=0.150, loss_ctc=1.700, loss=1.700, backward_time=0.100, optim_step_time=0.033, optim0_lr0=4.111e-04, train_time=1.214 +[ALab02] 2023-06-10 05:38:20,972 (trainer:721) INFO: 19epoch:train:8817-9367batch: iter_time=1.936e-04, forward_time=0.151, loss_ctc=1.593, loss=1.593, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.106e-04, train_time=1.216 +[ALab02] 2023-06-10 05:41:08,679 (trainer:721) INFO: 19epoch:train:9368-9918batch: iter_time=1.986e-04, forward_time=0.150, loss_ctc=1.644, loss=1.644, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.101e-04, train_time=1.219 +[ALab02] 2023-06-10 05:43:53,294 (trainer:721) INFO: 19epoch:train:9919-10469batch: iter_time=1.992e-04, forward_time=0.146, loss_ctc=1.637, loss=1.637, backward_time=0.100, optim_step_time=0.033, optim0_lr0=4.095e-04, train_time=1.193 +[ALab02] 2023-06-10 05:46:05,406 (trainer:721) INFO: 19epoch:train:10470-11020batch: iter_time=1.934e-04, forward_time=0.108, loss_ctc=1.641, loss=1.641, backward_time=0.090, optim_step_time=0.034, optim0_lr0=4.090e-04, train_time=0.959 +[ALab02] 2023-06-10 05:47:36,959 (trainer:338) INFO: 19epoch results: [train] iter_time=2.112e-04, forward_time=0.145, loss_ctc=1.654, loss=1.654, backward_time=0.098, optim_step_time=0.034, optim0_lr0=4.142e-04, train_time=1.185, time=54 minutes and 26.94 seconds, total_count=209551, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.153, cer_ctc=0.062, cer=0.062, loss=4.153, time=57.55 seconds, total_count=8379, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.97 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 05:47:39,371 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 05:47:39,373 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/9epoch.pth +[ALab02] 2023-06-10 05:47:39,373 (trainer:272) INFO: 20/70epoch started. Estimated time to finish: 1 day, 22 hours and 5 minutes +[ALab02] 2023-06-10 05:50:27,744 (trainer:721) INFO: 20epoch:train:1-551batch: iter_time=0.001, forward_time=0.151, loss_ctc=1.610, loss=1.610, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.084e-04, train_time=1.222 +[ALab02] 2023-06-10 05:53:17,113 (trainer:721) INFO: 20epoch:train:552-1102batch: iter_time=2.790e-04, forward_time=0.151, loss_ctc=1.616, loss=1.616, backward_time=0.101, optim_step_time=0.034, optim0_lr0=4.079e-04, train_time=1.229 +[ALab02] 2023-06-10 05:56:02,934 (trainer:721) INFO: 20epoch:train:1103-1653batch: iter_time=2.418e-04, forward_time=0.149, loss_ctc=1.580, loss=1.580, backward_time=0.098, optim_step_time=0.034, optim0_lr0=4.074e-04, train_time=1.204 +[ALab02] 2023-06-10 05:58:50,946 (trainer:721) INFO: 20epoch:train:1654-2204batch: iter_time=2.088e-04, forward_time=0.151, loss_ctc=1.582, loss=1.582, backward_time=0.100, optim_step_time=0.033, optim0_lr0=4.068e-04, train_time=1.220 +[ALab02] 2023-06-10 06:01:40,238 (trainer:721) INFO: 20epoch:train:2205-2755batch: iter_time=1.910e-04, forward_time=0.153, loss_ctc=1.504, loss=1.504, backward_time=0.101, optim_step_time=0.034, optim0_lr0=4.063e-04, train_time=1.232 +[ALab02] 2023-06-10 06:04:22,826 (trainer:721) INFO: 20epoch:train:2756-3306batch: iter_time=1.941e-04, forward_time=0.145, loss_ctc=1.632, loss=1.632, backward_time=0.098, optim_step_time=0.034, optim0_lr0=4.058e-04, train_time=1.179 +[ALab02] 2023-06-10 06:07:11,220 (trainer:721) INFO: 20epoch:train:3307-3857batch: iter_time=1.901e-04, forward_time=0.150, loss_ctc=1.653, loss=1.653, backward_time=0.101, optim_step_time=0.034, optim0_lr0=4.053e-04, train_time=1.222 +[ALab02] 2023-06-10 06:09:55,887 (trainer:721) INFO: 20epoch:train:3858-4408batch: iter_time=1.876e-04, forward_time=0.147, loss_ctc=1.680, loss=1.680, backward_time=0.098, optim_step_time=0.034, optim0_lr0=4.047e-04, train_time=1.194 +[ALab02] 2023-06-10 06:12:44,446 (trainer:721) INFO: 20epoch:train:4409-4959batch: iter_time=2.276e-04, forward_time=0.151, loss_ctc=1.735, loss=1.735, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.042e-04, train_time=1.223 +[ALab02] 2023-06-10 06:15:32,669 (trainer:721) INFO: 20epoch:train:4960-5510batch: iter_time=1.894e-04, forward_time=0.152, loss_ctc=1.571, loss=1.571, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.037e-04, train_time=1.221 +[ALab02] 2023-06-10 06:18:18,010 (trainer:721) INFO: 20epoch:train:5511-6061batch: iter_time=1.886e-04, forward_time=0.148, loss_ctc=1.638, loss=1.638, backward_time=0.099, optim_step_time=0.034, optim0_lr0=4.032e-04, train_time=1.200 +[ALab02] 2023-06-10 06:20:29,694 (trainer:721) INFO: 20epoch:train:6062-6612batch: iter_time=1.930e-04, forward_time=0.108, loss_ctc=1.643, loss=1.643, backward_time=0.089, optim_step_time=0.035, optim0_lr0=4.027e-04, train_time=0.957 +[ALab02] 2023-06-10 06:23:14,731 (trainer:721) INFO: 20epoch:train:6613-7163batch: iter_time=2.000e-04, forward_time=0.148, loss_ctc=1.586, loss=1.586, backward_time=0.099, optim_step_time=0.034, optim0_lr0=4.021e-04, train_time=1.198 +[ALab02] 2023-06-10 06:26:01,888 (trainer:721) INFO: 20epoch:train:7164-7714batch: iter_time=2.224e-04, forward_time=0.150, loss_ctc=1.598, loss=1.598, backward_time=0.099, optim_step_time=0.034, optim0_lr0=4.016e-04, train_time=1.212 +[ALab02] 2023-06-10 06:28:49,965 (trainer:721) INFO: 20epoch:train:7715-8265batch: iter_time=1.938e-04, forward_time=0.151, loss_ctc=1.626, loss=1.626, backward_time=0.100, optim_step_time=0.034, optim0_lr0=4.011e-04, train_time=1.221 +[ALab02] 2023-06-10 06:31:36,405 (trainer:721) INFO: 20epoch:train:8266-8816batch: iter_time=1.760e-04, forward_time=0.149, loss_ctc=1.663, loss=1.663, backward_time=0.099, optim_step_time=0.033, optim0_lr0=4.006e-04, train_time=1.208 +[ALab02] 2023-06-10 06:34:24,505 (trainer:721) INFO: 20epoch:train:8817-9367batch: iter_time=1.970e-04, forward_time=0.150, loss_ctc=1.616, loss=1.616, backward_time=0.101, optim_step_time=0.035, optim0_lr0=4.001e-04, train_time=1.219 +[ALab02] 2023-06-10 06:37:11,769 (trainer:721) INFO: 20epoch:train:9368-9918batch: iter_time=1.926e-04, forward_time=0.150, loss_ctc=1.667, loss=1.667, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.996e-04, train_time=1.215 +[ALab02] 2023-06-10 06:39:59,359 (trainer:721) INFO: 20epoch:train:9919-10469batch: iter_time=1.962e-04, forward_time=0.149, loss_ctc=1.618, loss=1.618, backward_time=0.101, optim_step_time=0.033, optim0_lr0=3.991e-04, train_time=1.217 +[ALab02] 2023-06-10 06:42:50,226 (trainer:721) INFO: 20epoch:train:10470-11020batch: iter_time=1.860e-04, forward_time=0.155, loss_ctc=1.568, loss=1.568, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.986e-04, train_time=1.241 +[ALab02] 2023-06-10 06:44:20,457 (trainer:338) INFO: 20epoch results: [train] iter_time=2.471e-04, forward_time=0.148, loss_ctc=1.619, loss=1.619, backward_time=0.099, optim_step_time=0.034, optim0_lr0=4.035e-04, train_time=1.202, time=55 minutes and 14.13 seconds, total_count=220580, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.159, cer_ctc=0.062, cer=0.062, loss=4.159, time=56.82 seconds, total_count=8820, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.13 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 06:44:22,792 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 06:44:22,794 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/10epoch.pth +[ALab02] 2023-06-10 06:44:22,794 (trainer:272) INFO: 21/70epoch started. Estimated time to finish: 1 day, 21 hours and 17 minutes +[ALab02] 2023-06-10 06:47:12,616 (trainer:721) INFO: 21epoch:train:1-551batch: iter_time=9.100e-04, forward_time=0.153, loss_ctc=1.544, loss=1.544, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.981e-04, train_time=1.232 +[ALab02] 2023-06-10 06:49:59,673 (trainer:721) INFO: 21epoch:train:552-1102batch: iter_time=2.144e-04, forward_time=0.149, loss_ctc=1.543, loss=1.543, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.976e-04, train_time=1.213 +[ALab02] 2023-06-10 06:52:45,423 (trainer:721) INFO: 21epoch:train:1103-1653batch: iter_time=1.952e-04, forward_time=0.147, loss_ctc=1.565, loss=1.565, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.971e-04, train_time=1.204 +[ALab02] 2023-06-10 06:54:55,504 (trainer:721) INFO: 21epoch:train:1654-2204batch: iter_time=1.693e-04, forward_time=0.107, loss_ctc=1.532, loss=1.532, backward_time=0.088, optim_step_time=0.034, optim0_lr0=3.966e-04, train_time=0.943 +[ALab02] 2023-06-10 06:58:08,005 (trainer:721) INFO: 21epoch:train:2205-2755batch: iter_time=2.851e-04, forward_time=0.170, loss_ctc=1.529, loss=1.529, backward_time=0.123, optim_step_time=0.048, optim0_lr0=3.961e-04, train_time=1.398 +[ALab02] 2023-06-10 07:00:56,223 (trainer:721) INFO: 21epoch:train:2756-3306batch: iter_time=1.855e-04, forward_time=0.151, loss_ctc=1.562, loss=1.562, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.956e-04, train_time=1.222 +[ALab02] 2023-06-10 07:03:42,920 (trainer:721) INFO: 21epoch:train:3307-3857batch: iter_time=1.982e-04, forward_time=0.150, loss_ctc=1.523, loss=1.523, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.952e-04, train_time=1.209 +[ALab02] 2023-06-10 07:06:29,085 (trainer:721) INFO: 21epoch:train:3858-4408batch: iter_time=1.904e-04, forward_time=0.148, loss_ctc=1.498, loss=1.498, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.947e-04, train_time=1.206 +[ALab02] 2023-06-10 07:09:17,022 (trainer:721) INFO: 21epoch:train:4409-4959batch: iter_time=1.992e-04, forward_time=0.152, loss_ctc=1.493, loss=1.493, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.942e-04, train_time=1.218 +[ALab02] 2023-06-10 07:12:04,813 (trainer:721) INFO: 21epoch:train:4960-5510batch: iter_time=1.983e-04, forward_time=0.151, loss_ctc=1.507, loss=1.507, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.937e-04, train_time=1.221 +[ALab02] 2023-06-10 07:14:53,252 (trainer:721) INFO: 21epoch:train:5511-6061batch: iter_time=1.812e-04, forward_time=0.151, loss_ctc=1.474, loss=1.474, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.932e-04, train_time=1.221 +[ALab02] 2023-06-10 07:17:40,985 (trainer:721) INFO: 21epoch:train:6062-6612batch: iter_time=2.202e-04, forward_time=0.149, loss_ctc=1.520, loss=1.520, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.927e-04, train_time=1.217 +[ALab02] 2023-06-10 07:20:30,198 (trainer:721) INFO: 21epoch:train:6613-7163batch: iter_time=2.017e-04, forward_time=0.152, loss_ctc=1.519, loss=1.519, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.923e-04, train_time=1.229 +[ALab02] 2023-06-10 07:23:17,882 (trainer:721) INFO: 21epoch:train:7164-7714batch: iter_time=2.037e-04, forward_time=0.150, loss_ctc=1.518, loss=1.518, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.918e-04, train_time=1.216 +[ALab02] 2023-06-10 07:26:06,129 (trainer:721) INFO: 21epoch:train:7715-8265batch: iter_time=1.872e-04, forward_time=0.150, loss_ctc=1.508, loss=1.508, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.913e-04, train_time=1.220 +[ALab02] 2023-06-10 07:28:46,694 (trainer:721) INFO: 21epoch:train:8266-8816batch: iter_time=1.747e-04, forward_time=0.143, loss_ctc=1.448, loss=1.448, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.909e-04, train_time=1.167 +[ALab02] 2023-06-10 07:31:03,669 (trainer:721) INFO: 21epoch:train:8817-9367batch: iter_time=1.770e-04, forward_time=0.116, loss_ctc=1.431, loss=1.431, backward_time=0.090, optim_step_time=0.033, optim0_lr0=3.904e-04, train_time=0.993 +[ALab02] 2023-06-10 07:33:50,445 (trainer:721) INFO: 21epoch:train:9368-9918batch: iter_time=1.766e-04, forward_time=0.151, loss_ctc=1.505, loss=1.505, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.899e-04, train_time=1.210 +[ALab02] 2023-06-10 07:36:37,224 (trainer:721) INFO: 21epoch:train:9919-10469batch: iter_time=1.664e-04, forward_time=0.149, loss_ctc=1.425, loss=1.425, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.895e-04, train_time=1.210 +[ALab02] 2023-06-10 07:39:22,055 (trainer:721) INFO: 21epoch:train:10470-11020batch: iter_time=1.755e-04, forward_time=0.146, loss_ctc=1.465, loss=1.465, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.890e-04, train_time=1.196 +[ALab02] 2023-06-10 07:40:52,240 (trainer:338) INFO: 21epoch results: [train] iter_time=2.304e-04, forward_time=0.147, loss_ctc=1.505, loss=1.505, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.935e-04, train_time=1.197, time=55 minutes and 2.38 seconds, total_count=231609, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.107, cer_ctc=0.061, cer=0.061, loss=4.107, time=57.66 seconds, total_count=9261, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.41 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 07:40:54,636 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 07:40:54,639 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/11epoch.pth +[ALab02] 2023-06-10 07:40:54,639 (trainer:272) INFO: 22/70epoch started. Estimated time to finish: 1 day, 20 hours and 28 minutes +[ALab02] 2023-06-10 07:43:39,242 (trainer:721) INFO: 22epoch:train:1-551batch: iter_time=7.177e-04, forward_time=0.146, loss_ctc=1.451, loss=1.451, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.885e-04, train_time=1.194 +[ALab02] 2023-06-10 07:46:26,510 (trainer:721) INFO: 22epoch:train:552-1102batch: iter_time=1.841e-04, forward_time=0.151, loss_ctc=1.432, loss=1.432, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.881e-04, train_time=1.214 +[ALab02] 2023-06-10 07:49:11,301 (trainer:721) INFO: 22epoch:train:1103-1653batch: iter_time=1.808e-04, forward_time=0.147, loss_ctc=1.428, loss=1.428, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.876e-04, train_time=1.197 +[ALab02] 2023-06-10 07:51:57,159 (trainer:721) INFO: 22epoch:train:1654-2204batch: iter_time=1.802e-04, forward_time=0.148, loss_ctc=1.488, loss=1.488, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.871e-04, train_time=1.203 +[ALab02] 2023-06-10 07:54:44,860 (trainer:721) INFO: 22epoch:train:2205-2755batch: iter_time=1.821e-04, forward_time=0.150, loss_ctc=1.432, loss=1.432, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.867e-04, train_time=1.217 +[ALab02] 2023-06-10 07:57:33,557 (trainer:721) INFO: 22epoch:train:2756-3306batch: iter_time=1.829e-04, forward_time=0.152, loss_ctc=1.507, loss=1.507, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.862e-04, train_time=1.224 +[ALab02] 2023-06-10 08:00:17,417 (trainer:721) INFO: 22epoch:train:3307-3857batch: iter_time=1.784e-04, forward_time=0.147, loss_ctc=1.447, loss=1.447, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.858e-04, train_time=1.190 +[ALab02] 2023-06-10 08:03:04,744 (trainer:721) INFO: 22epoch:train:3858-4408batch: iter_time=1.665e-04, forward_time=0.151, loss_ctc=1.422, loss=1.422, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.853e-04, train_time=1.214 +[ALab02] 2023-06-10 08:05:14,348 (trainer:721) INFO: 22epoch:train:4409-4959batch: iter_time=1.649e-04, forward_time=0.106, loss_ctc=1.452, loss=1.452, backward_time=0.088, optim_step_time=0.034, optim0_lr0=3.849e-04, train_time=0.939 +[ALab02] 2023-06-10 08:08:01,030 (trainer:721) INFO: 22epoch:train:4960-5510batch: iter_time=1.748e-04, forward_time=0.150, loss_ctc=1.412, loss=1.412, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.844e-04, train_time=1.211 +[ALab02] 2023-06-10 08:10:45,858 (trainer:721) INFO: 22epoch:train:5511-6061batch: iter_time=1.689e-04, forward_time=0.149, loss_ctc=1.443, loss=1.443, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.840e-04, train_time=1.196 +[ALab02] 2023-06-10 08:13:31,768 (trainer:721) INFO: 22epoch:train:6062-6612batch: iter_time=1.753e-04, forward_time=0.149, loss_ctc=1.410, loss=1.410, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.835e-04, train_time=1.205 +[ALab02] 2023-06-10 08:16:18,027 (trainer:721) INFO: 22epoch:train:6613-7163batch: iter_time=1.675e-04, forward_time=0.149, loss_ctc=1.411, loss=1.411, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.831e-04, train_time=1.208 +[ALab02] 2023-06-10 08:19:07,153 (trainer:721) INFO: 22epoch:train:7164-7714batch: iter_time=1.892e-04, forward_time=0.152, loss_ctc=1.355, loss=1.355, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.827e-04, train_time=1.227 +[ALab02] 2023-06-10 08:21:51,701 (trainer:721) INFO: 22epoch:train:7715-8265batch: iter_time=1.661e-04, forward_time=0.147, loss_ctc=1.424, loss=1.424, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.822e-04, train_time=1.195 +[ALab02] 2023-06-10 08:24:34,651 (trainer:721) INFO: 22epoch:train:8266-8816batch: iter_time=1.528e-04, forward_time=0.144, loss_ctc=1.448, loss=1.448, backward_time=0.098, optim_step_time=0.032, optim0_lr0=3.818e-04, train_time=1.182 +[ALab02] 2023-06-10 08:27:20,620 (trainer:721) INFO: 22epoch:train:8817-9367batch: iter_time=1.569e-04, forward_time=0.149, loss_ctc=1.455, loss=1.455, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.813e-04, train_time=1.206 +[ALab02] 2023-06-10 08:30:05,505 (trainer:721) INFO: 22epoch:train:9368-9918batch: iter_time=1.597e-04, forward_time=0.148, loss_ctc=1.447, loss=1.447, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.809e-04, train_time=1.196 +[ALab02] 2023-06-10 08:32:49,084 (trainer:721) INFO: 22epoch:train:9919-10469batch: iter_time=1.520e-04, forward_time=0.147, loss_ctc=1.417, loss=1.417, backward_time=0.097, optim_step_time=0.032, optim0_lr0=3.805e-04, train_time=1.187 +[ALab02] 2023-06-10 08:35:34,424 (trainer:721) INFO: 22epoch:train:10470-11020batch: iter_time=1.518e-04, forward_time=0.147, loss_ctc=1.410, loss=1.410, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.800e-04, train_time=1.200 +[ALab02] 2023-06-10 08:37:02,821 (trainer:338) INFO: 22epoch results: [train] iter_time=1.976e-04, forward_time=0.146, loss_ctc=1.434, loss=1.434, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.842e-04, train_time=1.190, time=54 minutes and 42.8 seconds, total_count=242638, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.080, cer_ctc=0.059, cer=0.059, loss=4.080, time=55.84 seconds, total_count=9702, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.53 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 08:37:05,218 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 08:37:05,235 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/12epoch.pth +[ALab02] 2023-06-10 08:37:05,235 (trainer:272) INFO: 23/70epoch started. Estimated time to finish: 1 day, 19 hours and 37 minutes +[ALab02] 2023-06-10 08:39:14,738 (trainer:721) INFO: 23epoch:train:1-551batch: iter_time=6.382e-04, forward_time=0.107, loss_ctc=1.384, loss=1.384, backward_time=0.087, optim_step_time=0.033, optim0_lr0=3.796e-04, train_time=0.938 +[ALab02] 2023-06-10 08:42:03,598 (trainer:721) INFO: 23epoch:train:552-1102batch: iter_time=1.737e-04, forward_time=0.152, loss_ctc=1.401, loss=1.401, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.792e-04, train_time=1.225 +[ALab02] 2023-06-10 08:44:48,051 (trainer:721) INFO: 23epoch:train:1103-1653batch: iter_time=1.676e-04, forward_time=0.147, loss_ctc=1.284, loss=1.284, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.787e-04, train_time=1.195 +[ALab02] 2023-06-10 08:47:37,131 (trainer:721) INFO: 23epoch:train:1654-2204batch: iter_time=1.739e-04, forward_time=0.153, loss_ctc=1.286, loss=1.286, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.783e-04, train_time=1.226 +[ALab02] 2023-06-10 08:50:23,671 (trainer:721) INFO: 23epoch:train:2205-2755batch: iter_time=1.753e-04, forward_time=0.149, loss_ctc=1.350, loss=1.350, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.779e-04, train_time=1.208 +[ALab02] 2023-06-10 08:53:11,286 (trainer:721) INFO: 23epoch:train:2756-3306batch: iter_time=1.750e-04, forward_time=0.151, loss_ctc=1.349, loss=1.349, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.775e-04, train_time=1.217 +[ALab02] 2023-06-10 08:55:59,069 (trainer:721) INFO: 23epoch:train:3307-3857batch: iter_time=1.772e-04, forward_time=0.150, loss_ctc=1.352, loss=1.352, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.770e-04, train_time=1.217 +[ALab02] 2023-06-10 08:58:46,624 (trainer:721) INFO: 23epoch:train:3858-4408batch: iter_time=1.806e-04, forward_time=0.151, loss_ctc=1.402, loss=1.402, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.766e-04, train_time=1.216 +[ALab02] 2023-06-10 09:01:33,162 (trainer:721) INFO: 23epoch:train:4409-4959batch: iter_time=1.688e-04, forward_time=0.148, loss_ctc=1.376, loss=1.376, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.762e-04, train_time=1.209 +[ALab02] 2023-06-10 09:04:21,029 (trainer:721) INFO: 23epoch:train:4960-5510batch: iter_time=1.667e-04, forward_time=0.151, loss_ctc=1.371, loss=1.371, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.758e-04, train_time=1.218 +[ALab02] 2023-06-10 09:07:05,276 (trainer:721) INFO: 23epoch:train:5511-6061batch: iter_time=1.714e-04, forward_time=0.148, loss_ctc=1.388, loss=1.388, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.754e-04, train_time=1.194 +[ALab02] 2023-06-10 09:09:53,504 (trainer:721) INFO: 23epoch:train:6062-6612batch: iter_time=1.816e-04, forward_time=0.151, loss_ctc=1.380, loss=1.380, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.749e-04, train_time=1.220 +[ALab02] 2023-06-10 09:12:38,527 (trainer:721) INFO: 23epoch:train:6613-7163batch: iter_time=1.717e-04, forward_time=0.147, loss_ctc=1.355, loss=1.355, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.745e-04, train_time=1.197 +[ALab02] 2023-06-10 09:14:48,027 (trainer:721) INFO: 23epoch:train:7164-7714batch: iter_time=1.801e-04, forward_time=0.107, loss_ctc=1.422, loss=1.422, backward_time=0.087, optim_step_time=0.034, optim0_lr0=3.741e-04, train_time=0.942 +[ALab02] 2023-06-10 09:17:33,375 (trainer:721) INFO: 23epoch:train:7715-8265batch: iter_time=1.684e-04, forward_time=0.148, loss_ctc=1.440, loss=1.440, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.737e-04, train_time=1.200 +[ALab02] 2023-06-10 09:20:19,121 (trainer:721) INFO: 23epoch:train:8266-8816batch: iter_time=1.656e-04, forward_time=0.148, loss_ctc=1.450, loss=1.450, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.733e-04, train_time=1.203 +[ALab02] 2023-06-10 09:23:02,309 (trainer:721) INFO: 23epoch:train:8817-9367batch: iter_time=1.679e-04, forward_time=0.147, loss_ctc=1.518, loss=1.518, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.729e-04, train_time=1.186 +[ALab02] 2023-06-10 09:25:48,601 (trainer:721) INFO: 23epoch:train:9368-9918batch: iter_time=1.750e-04, forward_time=0.149, loss_ctc=1.493, loss=1.493, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.725e-04, train_time=1.207 +[ALab02] 2023-06-10 09:28:36,195 (trainer:721) INFO: 23epoch:train:9919-10469batch: iter_time=1.767e-04, forward_time=0.150, loss_ctc=1.431, loss=1.431, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.721e-04, train_time=1.216 +[ALab02] 2023-06-10 09:31:22,068 (trainer:721) INFO: 23epoch:train:10470-11020batch: iter_time=1.718e-04, forward_time=0.150, loss_ctc=1.420, loss=1.420, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.717e-04, train_time=1.203 +[ALab02] 2023-06-10 09:32:53,341 (trainer:338) INFO: 23epoch results: [train] iter_time=1.963e-04, forward_time=0.145, loss_ctc=1.392, loss=1.392, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.756e-04, train_time=1.182, time=54 minutes and 20.09 seconds, total_count=253667, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.121, cer_ctc=0.059, cer=0.059, loss=4.121, time=57.43 seconds, total_count=10143, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.58 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 09:32:55,645 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 09:32:55,661 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/13epoch.pth +[ALab02] 2023-06-10 09:32:55,661 (trainer:272) INFO: 24/70epoch started. Estimated time to finish: 1 day, 18 hours and 45 minutes +[ALab02] 2023-06-10 09:35:44,158 (trainer:721) INFO: 24epoch:train:1-551batch: iter_time=0.002, forward_time=0.151, loss_ctc=1.368, loss=1.368, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.713e-04, train_time=1.223 +[ALab02] 2023-06-10 09:38:29,822 (trainer:721) INFO: 24epoch:train:552-1102batch: iter_time=2.414e-04, forward_time=0.148, loss_ctc=1.427, loss=1.427, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.709e-04, train_time=1.202 +[ALab02] 2023-06-10 09:41:15,840 (trainer:721) INFO: 24epoch:train:1103-1653batch: iter_time=1.960e-04, forward_time=0.149, loss_ctc=1.523, loss=1.523, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.705e-04, train_time=1.207 +[ALab02] 2023-06-10 09:44:02,662 (trainer:721) INFO: 24epoch:train:1654-2204batch: iter_time=1.728e-04, forward_time=0.149, loss_ctc=1.472, loss=1.472, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.701e-04, train_time=1.210 +[ALab02] 2023-06-10 09:46:47,476 (trainer:721) INFO: 24epoch:train:2205-2755batch: iter_time=1.719e-04, forward_time=0.148, loss_ctc=1.415, loss=1.415, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.697e-04, train_time=1.197 +[ALab02] 2023-06-10 09:49:01,126 (trainer:721) INFO: 24epoch:train:2756-3306batch: iter_time=1.682e-04, forward_time=0.110, loss_ctc=1.404, loss=1.404, backward_time=0.090, optim_step_time=0.035, optim0_lr0=3.693e-04, train_time=0.970 +[ALab02] 2023-06-10 09:51:46,701 (trainer:721) INFO: 24epoch:train:3307-3857batch: iter_time=1.848e-04, forward_time=0.148, loss_ctc=1.467, loss=1.467, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.689e-04, train_time=1.201 +[ALab02] 2023-06-10 09:54:34,108 (trainer:721) INFO: 24epoch:train:3858-4408batch: iter_time=1.699e-04, forward_time=0.150, loss_ctc=1.471, loss=1.471, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.685e-04, train_time=1.215 +[ALab02] 2023-06-10 09:57:20,418 (trainer:721) INFO: 24epoch:train:4409-4959batch: iter_time=1.830e-04, forward_time=0.149, loss_ctc=1.506, loss=1.506, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.681e-04, train_time=1.206 +[ALab02] 2023-06-10 10:00:08,217 (trainer:721) INFO: 24epoch:train:4960-5510batch: iter_time=1.775e-04, forward_time=0.151, loss_ctc=1.460, loss=1.460, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.677e-04, train_time=1.220 +[ALab02] 2023-06-10 10:02:53,999 (trainer:721) INFO: 24epoch:train:5511-6061batch: iter_time=1.613e-04, forward_time=0.149, loss_ctc=1.487, loss=1.487, backward_time=0.098, optim_step_time=0.032, optim0_lr0=3.673e-04, train_time=1.201 +[ALab02] 2023-06-10 10:05:39,298 (trainer:721) INFO: 24epoch:train:6062-6612batch: iter_time=1.928e-04, forward_time=0.148, loss_ctc=1.533, loss=1.533, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.669e-04, train_time=1.200 +[ALab02] 2023-06-10 10:08:23,058 (trainer:721) INFO: 24epoch:train:6613-7163batch: iter_time=1.690e-04, forward_time=0.146, loss_ctc=1.398, loss=1.398, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.665e-04, train_time=1.187 +[ALab02] 2023-06-10 10:11:08,564 (trainer:721) INFO: 24epoch:train:7164-7714batch: iter_time=1.511e-04, forward_time=0.150, loss_ctc=1.360, loss=1.360, backward_time=0.098, optim_step_time=0.032, optim0_lr0=3.661e-04, train_time=1.202 +[ALab02] 2023-06-10 10:13:52,037 (trainer:721) INFO: 24epoch:train:7715-8265batch: iter_time=1.575e-04, forward_time=0.146, loss_ctc=1.420, loss=1.420, backward_time=0.097, optim_step_time=0.032, optim0_lr0=3.657e-04, train_time=1.188 +[ALab02] 2023-06-10 10:16:37,370 (trainer:721) INFO: 24epoch:train:8266-8816batch: iter_time=1.515e-04, forward_time=0.149, loss_ctc=1.441, loss=1.441, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.654e-04, train_time=1.199 +[ALab02] 2023-06-10 10:19:22,855 (trainer:721) INFO: 24epoch:train:8817-9367batch: iter_time=1.534e-04, forward_time=0.147, loss_ctc=1.468, loss=1.468, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.650e-04, train_time=1.200 +[ALab02] 2023-06-10 10:22:06,290 (trainer:721) INFO: 24epoch:train:9368-9918batch: iter_time=1.525e-04, forward_time=0.147, loss_ctc=1.416, loss=1.416, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.646e-04, train_time=1.188 +[ALab02] 2023-06-10 10:24:16,316 (trainer:721) INFO: 24epoch:train:9919-10469batch: iter_time=1.571e-04, forward_time=0.107, loss_ctc=1.360, loss=1.360, backward_time=0.087, optim_step_time=0.033, optim0_lr0=3.642e-04, train_time=0.944 +[ALab02] 2023-06-10 10:27:04,102 (trainer:721) INFO: 24epoch:train:10470-11020batch: iter_time=1.986e-04, forward_time=0.150, loss_ctc=1.394, loss=1.394, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.638e-04, train_time=1.218 +[ALab02] 2023-06-10 10:28:33,561 (trainer:338) INFO: 24epoch results: [train] iter_time=2.800e-04, forward_time=0.145, loss_ctc=1.439, loss=1.439, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.675e-04, train_time=1.179, time=54 minutes and 11.71 seconds, total_count=264696, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.308, cer_ctc=0.059, cer=0.059, loss=4.308, time=56.73 seconds, total_count=10584, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.45 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 10:28:35,976 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 10:28:35,993 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/14epoch.pth +[ALab02] 2023-06-10 10:28:35,993 (trainer:272) INFO: 25/70epoch started. Estimated time to finish: 1 day, 17 hours and 53 minutes +[ALab02] 2023-06-10 10:31:19,329 (trainer:721) INFO: 25epoch:train:1-551batch: iter_time=6.036e-04, forward_time=0.145, loss_ctc=1.436, loss=1.436, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.635e-04, train_time=1.185 +[ALab02] 2023-06-10 10:34:04,834 (trainer:721) INFO: 25epoch:train:552-1102batch: iter_time=1.869e-04, forward_time=0.148, loss_ctc=1.382, loss=1.382, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.631e-04, train_time=1.203 +[ALab02] 2023-06-10 10:36:49,819 (trainer:721) INFO: 25epoch:train:1103-1653batch: iter_time=1.741e-04, forward_time=0.147, loss_ctc=1.295, loss=1.295, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.627e-04, train_time=1.196 +[ALab02] 2023-06-10 10:39:34,269 (trainer:721) INFO: 25epoch:train:1654-2204batch: iter_time=1.732e-04, forward_time=0.146, loss_ctc=1.336, loss=1.336, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.623e-04, train_time=1.193 +[ALab02] 2023-06-10 10:42:18,806 (trainer:721) INFO: 25epoch:train:2205-2755batch: iter_time=1.693e-04, forward_time=0.149, loss_ctc=1.255, loss=1.255, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.620e-04, train_time=1.195 +[ALab02] 2023-06-10 10:45:02,984 (trainer:721) INFO: 25epoch:train:2756-3306batch: iter_time=1.875e-04, forward_time=0.147, loss_ctc=1.321, loss=1.321, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.616e-04, train_time=1.191 +[ALab02] 2023-06-10 10:47:49,340 (trainer:721) INFO: 25epoch:train:3307-3857batch: iter_time=1.805e-04, forward_time=0.149, loss_ctc=1.293, loss=1.293, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.612e-04, train_time=1.208 +[ALab02] 2023-06-10 10:50:34,818 (trainer:721) INFO: 25epoch:train:3858-4408batch: iter_time=1.748e-04, forward_time=0.147, loss_ctc=1.367, loss=1.367, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.608e-04, train_time=1.200 +[ALab02] 2023-06-10 10:53:22,913 (trainer:721) INFO: 25epoch:train:4409-4959batch: iter_time=1.784e-04, forward_time=0.151, loss_ctc=1.330, loss=1.330, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.605e-04, train_time=1.221 +[ALab02] 2023-06-10 10:56:09,652 (trainer:721) INFO: 25epoch:train:4960-5510batch: iter_time=1.825e-04, forward_time=0.150, loss_ctc=1.284, loss=1.284, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.601e-04, train_time=1.210 +[ALab02] 2023-06-10 10:58:29,359 (trainer:721) INFO: 25epoch:train:5511-6061batch: iter_time=1.899e-04, forward_time=0.118, loss_ctc=1.292, loss=1.292, backward_time=0.092, optim_step_time=0.034, optim0_lr0=3.597e-04, train_time=1.014 +[ALab02] 2023-06-10 11:01:12,135 (trainer:721) INFO: 25epoch:train:6062-6612batch: iter_time=1.887e-04, forward_time=0.144, loss_ctc=1.329, loss=1.329, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.594e-04, train_time=1.180 +[ALab02] 2023-06-10 11:04:02,308 (trainer:721) INFO: 25epoch:train:6613-7163batch: iter_time=1.988e-04, forward_time=0.153, loss_ctc=1.305, loss=1.305, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.590e-04, train_time=1.234 +[ALab02] 2023-06-10 11:06:50,654 (trainer:721) INFO: 25epoch:train:7164-7714batch: iter_time=1.920e-04, forward_time=0.152, loss_ctc=1.336, loss=1.336, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.586e-04, train_time=1.223 +[ALab02] 2023-06-10 11:09:40,819 (trainer:721) INFO: 25epoch:train:7715-8265batch: iter_time=2.053e-04, forward_time=0.151, loss_ctc=1.306, loss=1.306, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.583e-04, train_time=1.235 +[ALab02] 2023-06-10 11:12:32,363 (trainer:721) INFO: 25epoch:train:8266-8816batch: iter_time=2.006e-04, forward_time=0.156, loss_ctc=1.255, loss=1.255, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.579e-04, train_time=1.244 +[ALab02] 2023-06-10 11:15:19,086 (trainer:721) INFO: 25epoch:train:8817-9367batch: iter_time=2.019e-04, forward_time=0.149, loss_ctc=1.266, loss=1.266, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.576e-04, train_time=1.211 +[ALab02] 2023-06-10 11:18:34,493 (trainer:721) INFO: 25epoch:train:9368-9918batch: iter_time=3.034e-04, forward_time=0.173, loss_ctc=1.253, loss=1.253, backward_time=0.125, optim_step_time=0.053, optim0_lr0=3.572e-04, train_time=1.416 +[ALab02] 2023-06-10 11:21:47,875 (trainer:721) INFO: 25epoch:train:9919-10469batch: iter_time=3.205e-04, forward_time=0.170, loss_ctc=1.248, loss=1.248, backward_time=0.124, optim_step_time=0.054, optim0_lr0=3.568e-04, train_time=1.403 +[ALab02] 2023-06-10 11:24:56,934 (trainer:721) INFO: 25epoch:train:10470-11020batch: iter_time=2.953e-04, forward_time=0.166, loss_ctc=1.310, loss=1.310, backward_time=0.120, optim_step_time=0.051, optim0_lr0=3.565e-04, train_time=1.372 +[ALab02] 2023-06-10 11:26:32,585 (trainer:338) INFO: 25epoch results: [train] iter_time=2.254e-04, forward_time=0.151, loss_ctc=1.309, loss=1.309, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.599e-04, train_time=1.227, time=56 minutes and 24.45 seconds, total_count=275725, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.012, cer_ctc=0.057, cer=0.057, loss=4.012, time=57.96 seconds, total_count=11025, gpu_max_cached_mem_GB=10.861, [att_plot] time=34.19 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 11:26:35,184 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 11:26:35,187 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/15epoch.pth +[ALab02] 2023-06-10 11:26:35,187 (trainer:272) INFO: 26/70epoch started. Estimated time to finish: 1 day, 17 hours and 4 minutes +[ALab02] 2023-06-10 11:29:28,831 (trainer:721) INFO: 26epoch:train:1-551batch: iter_time=0.002, forward_time=0.154, loss_ctc=1.200, loss=1.200, backward_time=0.106, optim_step_time=0.036, optim0_lr0=3.561e-04, train_time=1.261 +[ALab02] 2023-06-10 11:32:16,796 (trainer:721) INFO: 26epoch:train:552-1102batch: iter_time=2.148e-04, forward_time=0.151, loss_ctc=1.219, loss=1.219, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.558e-04, train_time=1.219 +[ALab02] 2023-06-10 11:34:28,398 (trainer:721) INFO: 26epoch:train:1103-1653batch: iter_time=1.949e-04, forward_time=0.106, loss_ctc=1.221, loss=1.221, backward_time=0.091, optim_step_time=0.035, optim0_lr0=3.554e-04, train_time=0.957 +[ALab02] 2023-06-10 11:37:18,728 (trainer:721) INFO: 26epoch:train:1654-2204batch: iter_time=1.995e-04, forward_time=0.153, loss_ctc=1.209, loss=1.209, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.551e-04, train_time=1.235 +[ALab02] 2023-06-10 11:40:05,906 (trainer:721) INFO: 26epoch:train:2205-2755batch: iter_time=1.936e-04, forward_time=0.149, loss_ctc=1.254, loss=1.254, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.547e-04, train_time=1.213 +[ALab02] 2023-06-10 11:42:55,221 (trainer:721) INFO: 26epoch:train:2756-3306batch: iter_time=1.908e-04, forward_time=0.152, loss_ctc=1.234, loss=1.234, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.544e-04, train_time=1.228 +[ALab02] 2023-06-10 11:45:44,743 (trainer:721) INFO: 26epoch:train:3307-3857batch: iter_time=2.013e-04, forward_time=0.154, loss_ctc=1.246, loss=1.246, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.540e-04, train_time=1.231 +[ALab02] 2023-06-10 11:48:31,452 (trainer:721) INFO: 26epoch:train:3858-4408batch: iter_time=1.828e-04, forward_time=0.150, loss_ctc=1.179, loss=1.179, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.537e-04, train_time=1.209 +[ALab02] 2023-06-10 11:51:24,827 (trainer:721) INFO: 26epoch:train:4409-4959batch: iter_time=2.276e-04, forward_time=0.155, loss_ctc=1.228, loss=1.228, backward_time=0.105, optim_step_time=0.037, optim0_lr0=3.533e-04, train_time=1.258 +[ALab02] 2023-06-10 11:54:12,221 (trainer:721) INFO: 26epoch:train:4960-5510batch: iter_time=1.984e-04, forward_time=0.149, loss_ctc=1.287, loss=1.287, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.530e-04, train_time=1.216 +[ALab02] 2023-06-10 11:56:59,960 (trainer:721) INFO: 26epoch:train:5511-6061batch: iter_time=1.980e-04, forward_time=0.148, loss_ctc=1.227, loss=1.227, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.526e-04, train_time=1.218 +[ALab02] 2023-06-10 11:59:49,028 (trainer:721) INFO: 26epoch:train:6062-6612batch: iter_time=2.008e-04, forward_time=0.153, loss_ctc=1.287, loss=1.287, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.523e-04, train_time=1.226 +[ALab02] 2023-06-10 12:02:36,578 (trainer:721) INFO: 26epoch:train:6613-7163batch: iter_time=1.906e-04, forward_time=0.150, loss_ctc=1.259, loss=1.259, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.519e-04, train_time=1.216 +[ALab02] 2023-06-10 12:05:25,027 (trainer:721) INFO: 26epoch:train:7164-7714batch: iter_time=1.936e-04, forward_time=0.152, loss_ctc=1.173, loss=1.173, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.516e-04, train_time=1.225 +[ALab02] 2023-06-10 12:07:57,961 (trainer:721) INFO: 26epoch:train:7715-8265batch: iter_time=1.889e-04, forward_time=0.131, loss_ctc=1.293, loss=1.293, backward_time=0.097, optim_step_time=0.034, optim0_lr0=3.512e-04, train_time=1.110 +[ALab02] 2023-06-10 12:10:24,547 (trainer:721) INFO: 26epoch:train:8266-8816batch: iter_time=1.801e-04, forward_time=0.126, loss_ctc=1.270, loss=1.270, backward_time=0.093, optim_step_time=0.035, optim0_lr0=3.509e-04, train_time=1.063 +[ALab02] 2023-06-10 12:13:12,087 (trainer:721) INFO: 26epoch:train:8817-9367batch: iter_time=2.053e-04, forward_time=0.150, loss_ctc=1.266, loss=1.266, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.506e-04, train_time=1.216 +[ALab02] 2023-06-10 12:15:58,473 (trainer:721) INFO: 26epoch:train:9368-9918batch: iter_time=1.942e-04, forward_time=0.149, loss_ctc=1.244, loss=1.244, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.502e-04, train_time=1.208 +[ALab02] 2023-06-10 12:18:45,858 (trainer:721) INFO: 26epoch:train:9919-10469batch: iter_time=1.874e-04, forward_time=0.151, loss_ctc=1.237, loss=1.237, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.499e-04, train_time=1.215 +[ALab02] 2023-06-10 12:21:32,911 (trainer:721) INFO: 26epoch:train:10470-11020batch: iter_time=2.016e-04, forward_time=0.148, loss_ctc=1.331, loss=1.331, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.496e-04, train_time=1.212 +[ALab02] 2023-06-10 12:23:03,148 (trainer:338) INFO: 26epoch results: [train] iter_time=2.635e-04, forward_time=0.147, loss_ctc=1.242, loss=1.242, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.528e-04, train_time=1.197, time=55 minutes and 0.87 seconds, total_count=286754, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.135, cer_ctc=0.057, cer=0.057, loss=4.135, time=56.51 seconds, total_count=11466, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.57 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 12:23:05,840 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 12:23:05,842 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/16epoch.pth +[ALab02] 2023-06-10 12:23:05,842 (trainer:272) INFO: 27/70epoch started. Estimated time to finish: 1 day, 16 hours and 12 minutes +[ALab02] 2023-06-10 12:25:54,833 (trainer:721) INFO: 27epoch:train:1-551batch: iter_time=7.005e-04, forward_time=0.151, loss_ctc=1.238, loss=1.238, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.492e-04, train_time=1.226 +[ALab02] 2023-06-10 12:28:41,771 (trainer:721) INFO: 27epoch:train:552-1102batch: iter_time=2.169e-04, forward_time=0.150, loss_ctc=1.206, loss=1.206, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.489e-04, train_time=1.213 +[ALab02] 2023-06-10 12:31:30,269 (trainer:721) INFO: 27epoch:train:1103-1653batch: iter_time=1.896e-04, forward_time=0.151, loss_ctc=1.192, loss=1.192, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.485e-04, train_time=1.222 +[ALab02] 2023-06-10 12:34:19,224 (trainer:721) INFO: 27epoch:train:1654-2204batch: iter_time=2.041e-04, forward_time=0.152, loss_ctc=1.210, loss=1.210, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.482e-04, train_time=1.226 +[ALab02] 2023-06-10 12:37:04,333 (trainer:721) INFO: 27epoch:train:2205-2755batch: iter_time=1.872e-04, forward_time=0.146, loss_ctc=1.227, loss=1.227, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.479e-04, train_time=1.200 +[ALab02] 2023-06-10 12:39:53,493 (trainer:721) INFO: 27epoch:train:2756-3306batch: iter_time=1.903e-04, forward_time=0.152, loss_ctc=1.177, loss=1.177, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.475e-04, train_time=1.225 +[ALab02] 2023-06-10 12:42:33,308 (trainer:721) INFO: 27epoch:train:3307-3857batch: iter_time=1.928e-04, forward_time=0.140, loss_ctc=1.198, loss=1.198, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.472e-04, train_time=1.162 +[ALab02] 2023-06-10 12:44:52,851 (trainer:721) INFO: 27epoch:train:3858-4408batch: iter_time=1.894e-04, forward_time=0.116, loss_ctc=1.198, loss=1.198, backward_time=0.092, optim_step_time=0.035, optim0_lr0=3.469e-04, train_time=1.012 +[ALab02] 2023-06-10 12:47:39,777 (trainer:721) INFO: 27epoch:train:4409-4959batch: iter_time=1.836e-04, forward_time=0.149, loss_ctc=1.191, loss=1.191, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.466e-04, train_time=1.211 +[ALab02] 2023-06-10 12:50:27,633 (trainer:721) INFO: 27epoch:train:4960-5510batch: iter_time=1.965e-04, forward_time=0.150, loss_ctc=1.191, loss=1.191, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.462e-04, train_time=1.217 +[ALab02] 2023-06-10 12:53:12,963 (trainer:721) INFO: 27epoch:train:5511-6061batch: iter_time=1.764e-04, forward_time=0.148, loss_ctc=1.197, loss=1.197, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.459e-04, train_time=1.202 +[ALab02] 2023-06-10 12:56:02,992 (trainer:721) INFO: 27epoch:train:6062-6612batch: iter_time=1.924e-04, forward_time=0.154, loss_ctc=1.201, loss=1.201, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.456e-04, train_time=1.234 +[ALab02] 2023-06-10 12:58:51,388 (trainer:721) INFO: 27epoch:train:6613-7163batch: iter_time=1.838e-04, forward_time=0.151, loss_ctc=1.202, loss=1.202, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.453e-04, train_time=1.221 +[ALab02] 2023-06-10 13:01:38,185 (trainer:721) INFO: 27epoch:train:7164-7714batch: iter_time=1.853e-04, forward_time=0.150, loss_ctc=1.198, loss=1.198, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.449e-04, train_time=1.213 +[ALab02] 2023-06-10 13:04:25,558 (trainer:721) INFO: 27epoch:train:7715-8265batch: iter_time=1.841e-04, forward_time=0.149, loss_ctc=1.229, loss=1.229, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.446e-04, train_time=1.214 +[ALab02] 2023-06-10 13:07:13,210 (trainer:721) INFO: 27epoch:train:8266-8816batch: iter_time=1.940e-04, forward_time=0.152, loss_ctc=1.165, loss=1.165, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.443e-04, train_time=1.217 +[ALab02] 2023-06-10 13:10:00,968 (trainer:721) INFO: 27epoch:train:8817-9367batch: iter_time=1.946e-04, forward_time=0.151, loss_ctc=1.179, loss=1.179, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.440e-04, train_time=1.217 +[ALab02] 2023-06-10 13:12:46,050 (trainer:721) INFO: 27epoch:train:9368-9918batch: iter_time=1.781e-04, forward_time=0.148, loss_ctc=1.200, loss=1.200, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.436e-04, train_time=1.198 +[ALab02] 2023-06-10 13:15:36,497 (trainer:721) INFO: 27epoch:train:9919-10469batch: iter_time=2.028e-04, forward_time=0.153, loss_ctc=1.283, loss=1.283, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.433e-04, train_time=1.238 +[ALab02] 2023-06-10 13:17:59,490 (trainer:721) INFO: 27epoch:train:10470-11020batch: iter_time=1.834e-04, forward_time=0.121, loss_ctc=1.275, loss=1.275, backward_time=0.092, optim_step_time=0.034, optim0_lr0=3.430e-04, train_time=1.038 +[ALab02] 2023-06-10 13:19:13,634 (trainer:338) INFO: 27epoch results: [train] iter_time=2.163e-04, forward_time=0.147, loss_ctc=1.208, loss=1.208, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.461e-04, train_time=1.195, time=54 minutes and 55.33 seconds, total_count=297783, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.165, cer_ctc=0.058, cer=0.058, loss=4.165, time=41.1 seconds, total_count=11907, gpu_max_cached_mem_GB=10.861, [att_plot] time=31.36 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 13:19:16,250 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 13:19:16,254 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/17epoch.pth +[ALab02] 2023-06-10 13:19:16,255 (trainer:272) INFO: 28/70epoch started. Estimated time to finish: 1 day, 15 hours and 20 minutes +[ALab02] 2023-06-10 13:22:03,306 (trainer:721) INFO: 28epoch:train:1-551batch: iter_time=7.318e-04, forward_time=0.150, loss_ctc=1.283, loss=1.283, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.427e-04, train_time=1.211 +[ALab02] 2023-06-10 13:24:49,666 (trainer:721) INFO: 28epoch:train:552-1102batch: iter_time=2.032e-04, forward_time=0.150, loss_ctc=1.201, loss=1.201, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.424e-04, train_time=1.208 +[ALab02] 2023-06-10 13:27:36,372 (trainer:721) INFO: 28epoch:train:1103-1653batch: iter_time=1.939e-04, forward_time=0.149, loss_ctc=1.273, loss=1.273, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.421e-04, train_time=1.210 +[ALab02] 2023-06-10 13:30:23,256 (trainer:721) INFO: 28epoch:train:1654-2204batch: iter_time=1.884e-04, forward_time=0.150, loss_ctc=1.230, loss=1.230, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.417e-04, train_time=1.212 +[ALab02] 2023-06-10 13:33:09,605 (trainer:721) INFO: 28epoch:train:2205-2755batch: iter_time=1.774e-04, forward_time=0.150, loss_ctc=1.347, loss=1.347, backward_time=0.098, optim_step_time=0.035, optim0_lr0=3.414e-04, train_time=1.206 +[ALab02] 2023-06-10 13:36:03,176 (trainer:721) INFO: 28epoch:train:2756-3306batch: iter_time=1.936e-04, forward_time=0.156, loss_ctc=1.261, loss=1.261, backward_time=0.104, optim_step_time=0.036, optim0_lr0=3.411e-04, train_time=1.263 +[ALab02] 2023-06-10 13:38:51,085 (trainer:721) INFO: 28epoch:train:3307-3857batch: iter_time=1.721e-04, forward_time=0.150, loss_ctc=1.251, loss=1.251, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.408e-04, train_time=1.216 +[ALab02] 2023-06-10 13:41:39,191 (trainer:721) INFO: 28epoch:train:3858-4408batch: iter_time=1.802e-04, forward_time=0.151, loss_ctc=1.253, loss=1.253, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.405e-04, train_time=1.220 +[ALab02] 2023-06-10 13:44:26,738 (trainer:721) INFO: 28epoch:train:4409-4959batch: iter_time=1.720e-04, forward_time=0.150, loss_ctc=1.288, loss=1.288, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.402e-04, train_time=1.217 +[ALab02] 2023-06-10 13:47:12,298 (trainer:721) INFO: 28epoch:train:4960-5510batch: iter_time=1.622e-04, forward_time=0.148, loss_ctc=1.274, loss=1.274, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.399e-04, train_time=1.202 +[ALab02] 2023-06-10 13:49:59,242 (trainer:721) INFO: 28epoch:train:5511-6061batch: iter_time=1.726e-04, forward_time=0.150, loss_ctc=1.305, loss=1.305, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.396e-04, train_time=1.212 +[ALab02] 2023-06-10 13:52:30,611 (trainer:721) INFO: 28epoch:train:6062-6612batch: iter_time=1.723e-04, forward_time=0.131, loss_ctc=1.324, loss=1.324, backward_time=0.094, optim_step_time=0.034, optim0_lr0=3.393e-04, train_time=1.098 +[ALab02] 2023-06-10 13:54:59,671 (trainer:721) INFO: 28epoch:train:6613-7163batch: iter_time=1.700e-04, forward_time=0.128, loss_ctc=1.268, loss=1.268, backward_time=0.096, optim_step_time=0.034, optim0_lr0=3.389e-04, train_time=1.081 +[ALab02] 2023-06-10 13:57:48,891 (trainer:721) INFO: 28epoch:train:7164-7714batch: iter_time=1.807e-04, forward_time=0.151, loss_ctc=1.276, loss=1.276, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.386e-04, train_time=1.228 +[ALab02] 2023-06-10 14:00:34,871 (trainer:721) INFO: 28epoch:train:7715-8265batch: iter_time=1.927e-04, forward_time=0.149, loss_ctc=1.344, loss=1.344, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.383e-04, train_time=1.205 +[ALab02] 2023-06-10 14:03:22,856 (trainer:721) INFO: 28epoch:train:8266-8816batch: iter_time=2.005e-04, forward_time=0.150, loss_ctc=1.275, loss=1.275, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.380e-04, train_time=1.219 +[ALab02] 2023-06-10 14:06:10,263 (trainer:721) INFO: 28epoch:train:8817-9367batch: iter_time=2.120e-04, forward_time=0.148, loss_ctc=1.313, loss=1.313, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.377e-04, train_time=1.214 +[ALab02] 2023-06-10 14:08:58,397 (trainer:721) INFO: 28epoch:train:9368-9918batch: iter_time=2.080e-04, forward_time=0.151, loss_ctc=1.374, loss=1.374, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.374e-04, train_time=1.223 +[ALab02] 2023-06-10 14:11:45,015 (trainer:721) INFO: 28epoch:train:9919-10469batch: iter_time=2.006e-04, forward_time=0.148, loss_ctc=1.395, loss=1.395, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.371e-04, train_time=1.207 +[ALab02] 2023-06-10 14:14:33,954 (trainer:721) INFO: 28epoch:train:10470-11020batch: iter_time=2.031e-04, forward_time=0.152, loss_ctc=1.354, loss=1.354, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.368e-04, train_time=1.226 +[ALab02] 2023-06-10 14:16:05,111 (trainer:338) INFO: 28epoch results: [train] iter_time=2.143e-04, forward_time=0.148, loss_ctc=1.294, loss=1.294, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.397e-04, train_time=1.204, time=55 minutes and 20.72 seconds, total_count=308812, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.252, cer_ctc=0.059, cer=0.059, loss=4.252, time=57.95 seconds, total_count=12348, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.18 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 14:16:07,595 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 14:16:07,596 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/18epoch.pth +[ALab02] 2023-06-10 14:16:07,596 (trainer:272) INFO: 29/70epoch started. Estimated time to finish: 1 day, 14 hours and 28 minutes +[ALab02] 2023-06-10 14:18:59,095 (trainer:721) INFO: 29epoch:train:1-551batch: iter_time=7.552e-04, forward_time=0.152, loss_ctc=1.330, loss=1.330, backward_time=0.104, optim_step_time=0.039, optim0_lr0=3.365e-04, train_time=1.248 +[ALab02] 2023-06-10 14:21:48,902 (trainer:721) INFO: 29epoch:train:552-1102batch: iter_time=2.074e-04, forward_time=0.153, loss_ctc=1.229, loss=1.229, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.362e-04, train_time=1.230 +[ALab02] 2023-06-10 14:24:36,674 (trainer:721) INFO: 29epoch:train:1103-1653batch: iter_time=2.084e-04, forward_time=0.150, loss_ctc=1.272, loss=1.272, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.359e-04, train_time=1.219 +[ALab02] 2023-06-10 14:27:15,665 (trainer:721) INFO: 29epoch:train:1654-2204batch: iter_time=2.021e-04, forward_time=0.139, loss_ctc=1.279, loss=1.279, backward_time=0.098, optim_step_time=0.035, optim0_lr0=3.356e-04, train_time=1.153 +[ALab02] 2023-06-10 14:29:33,715 (trainer:721) INFO: 29epoch:train:2205-2755batch: iter_time=2.092e-04, forward_time=0.115, loss_ctc=1.297, loss=1.297, backward_time=0.093, optim_step_time=0.037, optim0_lr0=3.353e-04, train_time=1.000 +[ALab02] 2023-06-10 14:32:22,937 (trainer:721) INFO: 29epoch:train:2756-3306batch: iter_time=2.065e-04, forward_time=0.150, loss_ctc=1.307, loss=1.307, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.350e-04, train_time=1.228 +[ALab02] 2023-06-10 14:35:08,030 (trainer:721) INFO: 29epoch:train:3307-3857batch: iter_time=2.073e-04, forward_time=0.147, loss_ctc=1.308, loss=1.308, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.347e-04, train_time=1.200 +[ALab02] 2023-06-10 14:37:56,340 (trainer:721) INFO: 29epoch:train:3858-4408batch: iter_time=1.926e-04, forward_time=0.151, loss_ctc=1.320, loss=1.320, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.344e-04, train_time=1.221 +[ALab02] 2023-06-10 14:40:44,072 (trainer:721) INFO: 29epoch:train:4409-4959batch: iter_time=2.007e-04, forward_time=0.151, loss_ctc=1.361, loss=1.361, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.341e-04, train_time=1.217 +[ALab02] 2023-06-10 14:43:32,156 (trainer:721) INFO: 29epoch:train:4960-5510batch: iter_time=2.164e-04, forward_time=0.151, loss_ctc=1.281, loss=1.281, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.339e-04, train_time=1.221 +[ALab02] 2023-06-10 14:46:17,989 (trainer:721) INFO: 29epoch:train:5511-6061batch: iter_time=2.039e-04, forward_time=0.149, loss_ctc=1.231, loss=1.231, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.336e-04, train_time=1.203 +[ALab02] 2023-06-10 14:49:07,158 (trainer:721) INFO: 29epoch:train:6062-6612batch: iter_time=2.051e-04, forward_time=0.150, loss_ctc=1.245, loss=1.245, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.333e-04, train_time=1.228 +[ALab02] 2023-06-10 14:51:57,882 (trainer:721) INFO: 29epoch:train:6613-7163batch: iter_time=2.073e-04, forward_time=0.154, loss_ctc=1.297, loss=1.297, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.330e-04, train_time=1.239 +[ALab02] 2023-06-10 14:54:43,511 (trainer:721) INFO: 29epoch:train:7164-7714batch: iter_time=2.023e-04, forward_time=0.148, loss_ctc=1.238, loss=1.238, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.327e-04, train_time=1.203 +[ALab02] 2023-06-10 14:57:34,504 (trainer:721) INFO: 29epoch:train:7715-8265batch: iter_time=1.873e-04, forward_time=0.153, loss_ctc=1.251, loss=1.251, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.324e-04, train_time=1.239 +[ALab02] 2023-06-10 15:00:20,998 (trainer:721) INFO: 29epoch:train:8266-8816batch: iter_time=2.002e-04, forward_time=0.149, loss_ctc=1.262, loss=1.262, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.321e-04, train_time=1.209 +[ALab02] 2023-06-10 15:02:47,700 (trainer:721) INFO: 29epoch:train:8817-9367batch: iter_time=2.014e-04, forward_time=0.125, loss_ctc=1.334, loss=1.334, backward_time=0.094, optim_step_time=0.035, optim0_lr0=3.318e-04, train_time=1.067 +[ALab02] 2023-06-10 15:05:23,612 (trainer:721) INFO: 29epoch:train:9368-9918batch: iter_time=1.991e-04, forward_time=0.136, loss_ctc=1.235, loss=1.235, backward_time=0.097, optim_step_time=0.036, optim0_lr0=3.315e-04, train_time=1.128 +[ALab02] 2023-06-10 15:08:15,365 (trainer:721) INFO: 29epoch:train:9919-10469batch: iter_time=2.152e-04, forward_time=0.152, loss_ctc=1.263, loss=1.263, backward_time=0.105, optim_step_time=0.038, optim0_lr0=3.312e-04, train_time=1.247 +[ALab02] 2023-06-10 15:11:09,015 (trainer:721) INFO: 29epoch:train:10470-11020batch: iter_time=2.281e-04, forward_time=0.157, loss_ctc=1.237, loss=1.237, backward_time=0.103, optim_step_time=0.038, optim0_lr0=3.310e-04, train_time=1.261 +[ALab02] 2023-06-10 15:12:42,138 (trainer:338) INFO: 29epoch results: [train] iter_time=2.327e-04, forward_time=0.147, loss_ctc=1.278, loss=1.278, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.337e-04, train_time=1.198, time=55 minutes and 4.59 seconds, total_count=319841, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.240, cer_ctc=0.058, cer=0.058, loss=4.240, time=56.76 seconds, total_count=12789, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.19 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 15:12:44,600 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 15:12:44,601 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/19epoch.pth +[ALab02] 2023-06-10 15:12:44,601 (trainer:272) INFO: 30/70epoch started. Estimated time to finish: 1 day, 13 hours and 35 minutes +[ALab02] 2023-06-10 15:15:36,363 (trainer:721) INFO: 30epoch:train:1-551batch: iter_time=0.001, forward_time=0.154, loss_ctc=1.188, loss=1.188, backward_time=0.103, optim_step_time=0.038, optim0_lr0=3.307e-04, train_time=1.247 +[ALab02] 2023-06-10 15:18:26,912 (trainer:721) INFO: 30epoch:train:552-1102batch: iter_time=3.587e-04, forward_time=0.152, loss_ctc=1.286, loss=1.286, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.304e-04, train_time=1.235 +[ALab02] 2023-06-10 15:21:21,602 (trainer:721) INFO: 30epoch:train:1103-1653batch: iter_time=3.008e-04, forward_time=0.155, loss_ctc=1.226, loss=1.226, backward_time=0.107, optim_step_time=0.041, optim0_lr0=3.301e-04, train_time=1.269 +[ALab02] 2023-06-10 15:24:13,984 (trainer:721) INFO: 30epoch:train:1654-2204batch: iter_time=2.239e-04, forward_time=0.155, loss_ctc=1.259, loss=1.259, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.298e-04, train_time=1.252 +[ALab02] 2023-06-10 15:27:03,466 (trainer:721) INFO: 30epoch:train:2205-2755batch: iter_time=2.245e-04, forward_time=0.150, loss_ctc=1.260, loss=1.260, backward_time=0.104, optim_step_time=0.038, optim0_lr0=3.295e-04, train_time=1.232 +[ALab02] 2023-06-10 15:29:54,186 (trainer:721) INFO: 30epoch:train:2756-3306batch: iter_time=2.249e-04, forward_time=0.154, loss_ctc=1.230, loss=1.230, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.293e-04, train_time=1.240 +[ALab02] 2023-06-10 15:32:44,867 (trainer:721) INFO: 30epoch:train:3307-3857batch: iter_time=2.381e-04, forward_time=0.153, loss_ctc=1.272, loss=1.272, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.290e-04, train_time=1.237 +[ALab02] 2023-06-10 15:35:33,639 (trainer:721) INFO: 30epoch:train:3858-4408batch: iter_time=2.205e-04, forward_time=0.151, loss_ctc=1.269, loss=1.269, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.287e-04, train_time=1.225 +[ALab02] 2023-06-10 15:37:47,032 (trainer:721) INFO: 30epoch:train:4409-4959batch: iter_time=2.185e-04, forward_time=0.108, loss_ctc=1.283, loss=1.283, backward_time=0.093, optim_step_time=0.037, optim0_lr0=3.284e-04, train_time=0.971 +[ALab02] 2023-06-10 15:40:39,540 (trainer:721) INFO: 30epoch:train:4960-5510batch: iter_time=2.109e-04, forward_time=0.156, loss_ctc=1.222, loss=1.222, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.281e-04, train_time=1.248 +[ALab02] 2023-06-10 15:43:28,737 (trainer:721) INFO: 30epoch:train:5511-6061batch: iter_time=2.077e-04, forward_time=0.152, loss_ctc=1.128, loss=1.128, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.279e-04, train_time=1.229 +[ALab02] 2023-06-10 15:46:15,813 (trainer:721) INFO: 30epoch:train:6062-6612batch: iter_time=2.181e-04, forward_time=0.147, loss_ctc=1.204, loss=1.204, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.276e-04, train_time=1.212 +[ALab02] 2023-06-10 15:49:09,174 (trainer:721) INFO: 30epoch:train:6613-7163batch: iter_time=2.173e-04, forward_time=0.156, loss_ctc=1.169, loss=1.169, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.273e-04, train_time=1.260 +[ALab02] 2023-06-10 15:52:00,387 (trainer:721) INFO: 30epoch:train:7164-7714batch: iter_time=2.193e-04, forward_time=0.153, loss_ctc=1.224, loss=1.224, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.270e-04, train_time=1.242 +[ALab02] 2023-06-10 15:54:52,243 (trainer:721) INFO: 30epoch:train:7715-8265batch: iter_time=2.265e-04, forward_time=0.153, loss_ctc=1.201, loss=1.201, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.268e-04, train_time=1.247 +[ALab02] 2023-06-10 15:57:44,142 (trainer:721) INFO: 30epoch:train:8266-8816batch: iter_time=2.223e-04, forward_time=0.154, loss_ctc=1.189, loss=1.189, backward_time=0.103, optim_step_time=0.038, optim0_lr0=3.265e-04, train_time=1.247 +[ALab02] 2023-06-10 16:00:38,467 (trainer:721) INFO: 30epoch:train:8817-9367batch: iter_time=2.174e-04, forward_time=0.158, loss_ctc=1.212, loss=1.212, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.262e-04, train_time=1.266 +[ALab02] 2023-06-10 16:03:30,120 (trainer:721) INFO: 30epoch:train:9368-9918batch: iter_time=2.071e-04, forward_time=0.154, loss_ctc=1.227, loss=1.227, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.259e-04, train_time=1.247 +[ALab02] 2023-06-10 16:06:19,199 (trainer:721) INFO: 30epoch:train:9919-10469batch: iter_time=2.087e-04, forward_time=0.153, loss_ctc=1.216, loss=1.216, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.257e-04, train_time=1.226 +[ALab02] 2023-06-10 16:09:06,587 (trainer:721) INFO: 30epoch:train:10470-11020batch: iter_time=2.151e-04, forward_time=0.148, loss_ctc=1.283, loss=1.283, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.254e-04, train_time=1.214 +[ALab02] 2023-06-10 16:10:38,621 (trainer:338) INFO: 30epoch results: [train] iter_time=2.827e-04, forward_time=0.151, loss_ctc=1.227, loss=1.227, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.280e-04, train_time=1.227, time=56 minutes and 25.12 seconds, total_count=330870, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.233, cer_ctc=0.056, cer=0.056, loss=4.233, time=57.21 seconds, total_count=13230, gpu_max_cached_mem_GB=10.861, [att_plot] time=31.69 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 16:10:41,097 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 16:10:41,099 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/20epoch.pth +[ALab02] 2023-06-10 16:10:41,099 (trainer:272) INFO: 31/70epoch started. Estimated time to finish: 1 day, 12 hours and 44 minutes +[ALab02] 2023-06-10 16:12:56,920 (trainer:721) INFO: 31epoch:train:1-551batch: iter_time=9.553e-04, forward_time=0.110, loss_ctc=1.218, loss=1.218, backward_time=0.093, optim_step_time=0.037, optim0_lr0=3.251e-04, train_time=0.986 +[ALab02] 2023-06-10 16:15:46,069 (trainer:721) INFO: 31epoch:train:552-1102batch: iter_time=2.250e-04, forward_time=0.151, loss_ctc=1.208, loss=1.208, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.248e-04, train_time=1.227 +[ALab02] 2023-06-10 16:18:32,727 (trainer:721) INFO: 31epoch:train:1103-1653batch: iter_time=2.107e-04, forward_time=0.149, loss_ctc=1.230, loss=1.230, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.246e-04, train_time=1.208 +[ALab02] 2023-06-10 16:21:21,542 (trainer:721) INFO: 31epoch:train:1654-2204batch: iter_time=2.094e-04, forward_time=0.151, loss_ctc=1.206, loss=1.206, backward_time=0.101, optim_step_time=0.037, optim0_lr0=3.243e-04, train_time=1.226 +[ALab02] 2023-06-10 16:24:10,042 (trainer:721) INFO: 31epoch:train:2205-2755batch: iter_time=2.116e-04, forward_time=0.150, loss_ctc=1.199, loss=1.199, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.240e-04, train_time=1.222 +[ALab02] 2023-06-10 16:27:02,081 (trainer:721) INFO: 31epoch:train:2756-3306batch: iter_time=2.116e-04, forward_time=0.155, loss_ctc=1.202, loss=1.202, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.238e-04, train_time=1.250 +[ALab02] 2023-06-10 16:29:53,920 (trainer:721) INFO: 31epoch:train:3307-3857batch: iter_time=2.102e-04, forward_time=0.156, loss_ctc=1.213, loss=1.213, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.235e-04, train_time=1.248 +[ALab02] 2023-06-10 16:32:44,645 (trainer:721) INFO: 31epoch:train:3858-4408batch: iter_time=2.163e-04, forward_time=0.152, loss_ctc=1.192, loss=1.192, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.232e-04, train_time=1.238 +[ALab02] 2023-06-10 16:35:34,608 (trainer:721) INFO: 31epoch:train:4409-4959batch: iter_time=2.109e-04, forward_time=0.152, loss_ctc=1.227, loss=1.227, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.230e-04, train_time=1.234 +[ALab02] 2023-06-10 16:38:23,721 (trainer:721) INFO: 31epoch:train:4960-5510batch: iter_time=2.036e-04, forward_time=0.151, loss_ctc=1.156, loss=1.156, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.227e-04, train_time=1.229 +[ALab02] 2023-06-10 16:41:09,719 (trainer:721) INFO: 31epoch:train:5511-6061batch: iter_time=2.215e-04, forward_time=0.149, loss_ctc=1.222, loss=1.222, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.224e-04, train_time=1.204 +[ALab02] 2023-06-10 16:44:02,479 (trainer:721) INFO: 31epoch:train:6062-6612batch: iter_time=2.215e-04, forward_time=0.155, loss_ctc=1.211, loss=1.211, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.222e-04, train_time=1.253 +[ALab02] 2023-06-10 16:46:50,088 (trainer:721) INFO: 31epoch:train:6613-7163batch: iter_time=2.077e-04, forward_time=0.150, loss_ctc=1.173, loss=1.173, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.219e-04, train_time=1.219 +[ALab02] 2023-06-10 16:49:07,251 (trainer:721) INFO: 31epoch:train:7164-7714batch: iter_time=2.065e-04, forward_time=0.113, loss_ctc=1.175, loss=1.175, backward_time=0.093, optim_step_time=0.037, optim0_lr0=3.217e-04, train_time=0.994 +[ALab02] 2023-06-10 16:51:59,180 (trainer:721) INFO: 31epoch:train:7715-8265batch: iter_time=2.224e-04, forward_time=0.154, loss_ctc=1.206, loss=1.206, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.214e-04, train_time=1.248 +[ALab02] 2023-06-10 16:54:49,687 (trainer:721) INFO: 31epoch:train:8266-8816batch: iter_time=2.219e-04, forward_time=0.153, loss_ctc=1.148, loss=1.148, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.211e-04, train_time=1.237 +[ALab02] 2023-06-10 16:57:41,791 (trainer:721) INFO: 31epoch:train:8817-9367batch: iter_time=2.161e-04, forward_time=0.154, loss_ctc=1.105, loss=1.105, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.209e-04, train_time=1.250 +[ALab02] 2023-06-10 17:00:28,183 (trainer:721) INFO: 31epoch:train:9368-9918batch: iter_time=2.095e-04, forward_time=0.147, loss_ctc=1.163, loss=1.163, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.206e-04, train_time=1.209 +[ALab02] 2023-06-10 17:03:15,537 (trainer:721) INFO: 31epoch:train:9919-10469batch: iter_time=2.040e-04, forward_time=0.148, loss_ctc=1.165, loss=1.165, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.204e-04, train_time=1.214 +[ALab02] 2023-06-10 17:06:05,113 (trainer:721) INFO: 31epoch:train:10470-11020batch: iter_time=2.017e-04, forward_time=0.151, loss_ctc=1.149, loss=1.149, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.201e-04, train_time=1.230 +[ALab02] 2023-06-10 17:07:36,546 (trainer:338) INFO: 31epoch results: [train] iter_time=2.498e-04, forward_time=0.148, loss_ctc=1.188, loss=1.188, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.226e-04, train_time=1.206, time=55 minutes and 27.35 seconds, total_count=341899, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.202, cer_ctc=0.057, cer=0.057, loss=4.202, time=57.02 seconds, total_count=13671, gpu_max_cached_mem_GB=10.861, [att_plot] time=31.07 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 17:07:38,772 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 17:07:38,773 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/21epoch.pth +[ALab02] 2023-06-10 17:07:38,774 (trainer:272) INFO: 32/70epoch started. Estimated time to finish: 1 day, 11 hours and 51 minutes +[ALab02] 2023-06-10 17:10:27,854 (trainer:721) INFO: 32epoch:train:1-551batch: iter_time=0.001, forward_time=0.151, loss_ctc=1.149, loss=1.149, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.198e-04, train_time=1.227 +[ALab02] 2023-06-10 17:13:16,003 (trainer:721) INFO: 32epoch:train:552-1102batch: iter_time=2.637e-04, forward_time=0.150, loss_ctc=1.155, loss=1.155, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.196e-04, train_time=1.221 +[ALab02] 2023-06-10 17:16:05,120 (trainer:721) INFO: 32epoch:train:1103-1653batch: iter_time=2.153e-04, forward_time=0.152, loss_ctc=1.088, loss=1.088, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.193e-04, train_time=1.226 +[ALab02] 2023-06-10 17:18:55,689 (trainer:721) INFO: 32epoch:train:1654-2204batch: iter_time=2.159e-04, forward_time=0.152, loss_ctc=1.112, loss=1.112, backward_time=0.103, optim_step_time=0.037, optim0_lr0=3.191e-04, train_time=1.238 +[ALab02] 2023-06-10 17:21:37,421 (trainer:721) INFO: 32epoch:train:2205-2755batch: iter_time=2.182e-04, forward_time=0.142, loss_ctc=1.084, loss=1.084, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.188e-04, train_time=1.177 +[ALab02] 2023-06-10 17:23:59,508 (trainer:721) INFO: 32epoch:train:2756-3306batch: iter_time=2.205e-04, forward_time=0.120, loss_ctc=1.132, loss=1.132, backward_time=0.093, optim_step_time=0.036, optim0_lr0=3.186e-04, train_time=1.027 +[ALab02] 2023-06-10 17:26:50,030 (trainer:721) INFO: 32epoch:train:3307-3857batch: iter_time=2.071e-04, forward_time=0.151, loss_ctc=1.102, loss=1.102, backward_time=0.103, optim_step_time=0.035, optim0_lr0=3.183e-04, train_time=1.239 +[ALab02] 2023-06-10 17:29:41,362 (trainer:721) INFO: 32epoch:train:3858-4408batch: iter_time=2.183e-04, forward_time=0.154, loss_ctc=1.175, loss=1.175, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.180e-04, train_time=1.243 +[ALab02] 2023-06-10 17:32:28,778 (trainer:721) INFO: 32epoch:train:4409-4959batch: iter_time=2.325e-04, forward_time=0.148, loss_ctc=1.166, loss=1.166, backward_time=0.101, optim_step_time=0.037, optim0_lr0=3.178e-04, train_time=1.215 +[ALab02] 2023-06-10 17:35:20,389 (trainer:721) INFO: 32epoch:train:4960-5510batch: iter_time=2.105e-04, forward_time=0.154, loss_ctc=1.139, loss=1.139, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.175e-04, train_time=1.245 +[ALab02] 2023-06-10 17:38:09,276 (trainer:721) INFO: 32epoch:train:5511-6061batch: iter_time=2.315e-04, forward_time=0.150, loss_ctc=1.148, loss=1.148, backward_time=0.102, optim_step_time=0.037, optim0_lr0=3.173e-04, train_time=1.226 +[ALab02] 2023-06-10 17:40:58,166 (trainer:721) INFO: 32epoch:train:6062-6612batch: iter_time=2.134e-04, forward_time=0.151, loss_ctc=1.154, loss=1.154, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.170e-04, train_time=1.226 +[ALab02] 2023-06-10 17:43:48,618 (trainer:721) INFO: 32epoch:train:6613-7163batch: iter_time=2.019e-04, forward_time=0.154, loss_ctc=1.213, loss=1.213, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.168e-04, train_time=1.238 +[ALab02] 2023-06-10 17:46:39,071 (trainer:721) INFO: 32epoch:train:7164-7714batch: iter_time=2.135e-04, forward_time=0.153, loss_ctc=1.195, loss=1.195, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.165e-04, train_time=1.237 +[ALab02] 2023-06-10 17:49:28,888 (trainer:721) INFO: 32epoch:train:7715-8265batch: iter_time=2.079e-04, forward_time=0.151, loss_ctc=1.155, loss=1.155, backward_time=0.101, optim_step_time=0.036, optim0_lr0=3.163e-04, train_time=1.233 +[ALab02] 2023-06-10 17:52:16,554 (trainer:721) INFO: 32epoch:train:8266-8816batch: iter_time=2.225e-04, forward_time=0.149, loss_ctc=1.191, loss=1.191, backward_time=0.101, optim_step_time=0.037, optim0_lr0=3.160e-04, train_time=1.216 +[ALab02] 2023-06-10 17:55:05,113 (trainer:721) INFO: 32epoch:train:8817-9367batch: iter_time=2.197e-04, forward_time=0.151, loss_ctc=1.150, loss=1.150, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.158e-04, train_time=1.223 +[ALab02] 2023-06-10 17:57:21,445 (trainer:721) INFO: 32epoch:train:9368-9918batch: iter_time=2.130e-04, forward_time=0.113, loss_ctc=1.235, loss=1.235, backward_time=0.091, optim_step_time=0.036, optim0_lr0=3.155e-04, train_time=0.991 +[ALab02] 2023-06-10 18:00:08,065 (trainer:721) INFO: 32epoch:train:9919-10469batch: iter_time=2.043e-04, forward_time=0.150, loss_ctc=1.179, loss=1.179, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.153e-04, train_time=1.209 +[ALab02] 2023-06-10 18:02:53,453 (trainer:721) INFO: 32epoch:train:10470-11020batch: iter_time=2.093e-04, forward_time=0.147, loss_ctc=1.146, loss=1.146, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.151e-04, train_time=1.201 +[ALab02] 2023-06-10 18:04:24,473 (trainer:338) INFO: 32epoch results: [train] iter_time=2.610e-04, forward_time=0.147, loss_ctc=1.153, loss=1.153, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.174e-04, train_time=1.203, time=55 minutes and 18.13 seconds, total_count=352928, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.287, cer_ctc=0.057, cer=0.057, loss=4.287, time=56.74 seconds, total_count=14112, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.83 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 18:04:26,900 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 18:04:26,901 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/22epoch.pth +[ALab02] 2023-06-10 18:04:26,902 (trainer:272) INFO: 33/70epoch started. Estimated time to finish: 1 day, 10 hours and 58 minutes +[ALab02] 2023-06-10 18:07:16,995 (trainer:721) INFO: 33epoch:train:1-551batch: iter_time=6.670e-04, forward_time=0.152, loss_ctc=1.091, loss=1.091, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.148e-04, train_time=1.235 +[ALab02] 2023-06-10 18:10:07,449 (trainer:721) INFO: 33epoch:train:552-1102batch: iter_time=2.095e-04, forward_time=0.153, loss_ctc=1.126, loss=1.126, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.146e-04, train_time=1.236 +[ALab02] 2023-06-10 18:12:58,018 (trainer:721) INFO: 33epoch:train:1103-1653batch: iter_time=1.994e-04, forward_time=0.153, loss_ctc=1.071, loss=1.071, backward_time=0.102, optim_step_time=0.034, optim0_lr0=3.143e-04, train_time=1.238 +[ALab02] 2023-06-10 18:15:46,677 (trainer:721) INFO: 33epoch:train:1654-2204batch: iter_time=1.927e-04, forward_time=0.152, loss_ctc=1.082, loss=1.082, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.141e-04, train_time=1.224 +[ALab02] 2023-06-10 18:18:36,259 (trainer:721) INFO: 33epoch:train:2205-2755batch: iter_time=1.939e-04, forward_time=0.151, loss_ctc=1.142, loss=1.142, backward_time=0.102, optim_step_time=0.036, optim0_lr0=3.138e-04, train_time=1.231 +[ALab02] 2023-06-10 18:21:23,617 (trainer:721) INFO: 33epoch:train:2756-3306batch: iter_time=2.000e-04, forward_time=0.149, loss_ctc=1.155, loss=1.155, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.136e-04, train_time=1.216 +[ALab02] 2023-06-10 18:24:11,694 (trainer:721) INFO: 33epoch:train:3307-3857batch: iter_time=2.061e-04, forward_time=0.151, loss_ctc=1.140, loss=1.140, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.133e-04, train_time=1.219 +[ALab02] 2023-06-10 18:26:58,351 (trainer:721) INFO: 33epoch:train:3858-4408batch: iter_time=1.904e-04, forward_time=0.150, loss_ctc=1.125, loss=1.125, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.131e-04, train_time=1.210 +[ALab02] 2023-06-10 18:29:48,656 (trainer:721) INFO: 33epoch:train:4409-4959batch: iter_time=1.893e-04, forward_time=0.154, loss_ctc=1.120, loss=1.120, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.129e-04, train_time=1.235 +[ALab02] 2023-06-10 18:32:00,111 (trainer:721) INFO: 33epoch:train:4960-5510batch: iter_time=2.188e-04, forward_time=0.107, loss_ctc=1.135, loss=1.135, backward_time=0.089, optim_step_time=0.034, optim0_lr0=3.126e-04, train_time=0.954 +[ALab02] 2023-06-10 18:34:49,905 (trainer:721) INFO: 33epoch:train:5511-6061batch: iter_time=2.333e-04, forward_time=0.153, loss_ctc=1.150, loss=1.150, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.124e-04, train_time=1.233 +[ALab02] 2023-06-10 18:37:37,146 (trainer:721) INFO: 33epoch:train:6062-6612batch: iter_time=2.021e-04, forward_time=0.150, loss_ctc=1.162, loss=1.162, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.121e-04, train_time=1.215 +[ALab02] 2023-06-10 18:40:25,925 (trainer:721) INFO: 33epoch:train:6613-7163batch: iter_time=2.024e-04, forward_time=0.152, loss_ctc=1.144, loss=1.144, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.119e-04, train_time=1.225 +[ALab02] 2023-06-10 18:43:15,348 (trainer:721) INFO: 33epoch:train:7164-7714batch: iter_time=2.059e-04, forward_time=0.151, loss_ctc=1.099, loss=1.099, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.117e-04, train_time=1.229 +[ALab02] 2023-06-10 18:46:04,716 (trainer:721) INFO: 33epoch:train:7715-8265batch: iter_time=2.052e-04, forward_time=0.153, loss_ctc=1.089, loss=1.089, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.114e-04, train_time=1.230 +[ALab02] 2023-06-10 18:48:48,486 (trainer:721) INFO: 33epoch:train:8266-8816batch: iter_time=1.829e-04, forward_time=0.145, loss_ctc=1.139, loss=1.139, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.112e-04, train_time=1.189 +[ALab02] 2023-06-10 18:51:38,238 (trainer:721) INFO: 33epoch:train:8817-9367batch: iter_time=2.079e-04, forward_time=0.150, loss_ctc=1.112, loss=1.112, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.109e-04, train_time=1.232 +[ALab02] 2023-06-10 18:54:26,092 (trainer:721) INFO: 33epoch:train:9368-9918batch: iter_time=1.903e-04, forward_time=0.150, loss_ctc=1.131, loss=1.131, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.107e-04, train_time=1.219 +[ALab02] 2023-06-10 18:57:12,766 (trainer:721) INFO: 33epoch:train:9919-10469batch: iter_time=1.800e-04, forward_time=0.151, loss_ctc=1.074, loss=1.074, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.105e-04, train_time=1.210 +[ALab02] 2023-06-10 18:59:59,376 (trainer:721) INFO: 33epoch:train:10470-11020batch: iter_time=1.870e-04, forward_time=0.149, loss_ctc=1.103, loss=1.103, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.102e-04, train_time=1.208 +[ALab02] 2023-06-10 19:01:29,589 (trainer:338) INFO: 33epoch results: [train] iter_time=2.231e-04, forward_time=0.149, loss_ctc=1.120, loss=1.120, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.125e-04, train_time=1.210, time=55 minutes and 35.92 seconds, total_count=363957, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.302, cer_ctc=0.056, cer=0.056, loss=4.302, time=56.5 seconds, total_count=14553, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.26 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 19:01:32,136 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 19:01:32,138 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/24epoch.pth +[ALab02] 2023-06-10 19:01:32,138 (trainer:272) INFO: 34/70epoch started. Estimated time to finish: 1 day, 10 hours and 5 minutes +[ALab02] 2023-06-10 19:04:18,460 (trainer:721) INFO: 34epoch:train:1-551batch: iter_time=8.389e-04, forward_time=0.149, loss_ctc=1.076, loss=1.076, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.100e-04, train_time=1.208 +[ALab02] 2023-06-10 19:06:29,987 (trainer:721) INFO: 34epoch:train:552-1102batch: iter_time=1.898e-04, forward_time=0.108, loss_ctc=1.089, loss=1.089, backward_time=0.089, optim_step_time=0.033, optim0_lr0=3.098e-04, train_time=0.955 +[ALab02] 2023-06-10 19:09:15,178 (trainer:721) INFO: 34epoch:train:1103-1653batch: iter_time=1.839e-04, forward_time=0.148, loss_ctc=1.057, loss=1.057, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.095e-04, train_time=1.199 +[ALab02] 2023-06-10 19:12:00,024 (trainer:721) INFO: 34epoch:train:1654-2204batch: iter_time=2.113e-04, forward_time=0.147, loss_ctc=1.070, loss=1.070, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.093e-04, train_time=1.196 +[ALab02] 2023-06-10 19:14:44,516 (trainer:721) INFO: 34epoch:train:2205-2755batch: iter_time=1.945e-04, forward_time=0.147, loss_ctc=1.082, loss=1.082, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.091e-04, train_time=1.194 +[ALab02] 2023-06-10 19:17:32,539 (trainer:721) INFO: 34epoch:train:2756-3306batch: iter_time=1.977e-04, forward_time=0.152, loss_ctc=1.054, loss=1.054, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.088e-04, train_time=1.220 +[ALab02] 2023-06-10 19:20:20,550 (trainer:721) INFO: 34epoch:train:3307-3857batch: iter_time=1.971e-04, forward_time=0.151, loss_ctc=1.052, loss=1.052, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.086e-04, train_time=1.217 +[ALab02] 2023-06-10 19:23:07,716 (trainer:721) INFO: 34epoch:train:3858-4408batch: iter_time=1.914e-04, forward_time=0.150, loss_ctc=1.043, loss=1.043, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.084e-04, train_time=1.215 +[ALab02] 2023-06-10 19:25:56,775 (trainer:721) INFO: 34epoch:train:4409-4959batch: iter_time=1.954e-04, forward_time=0.153, loss_ctc=1.017, loss=1.017, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.081e-04, train_time=1.228 +[ALab02] 2023-06-10 19:28:44,361 (trainer:721) INFO: 34epoch:train:4960-5510batch: iter_time=2.005e-04, forward_time=0.150, loss_ctc=1.066, loss=1.066, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.079e-04, train_time=1.217 +[ALab02] 2023-06-10 19:31:31,705 (trainer:721) INFO: 34epoch:train:5511-6061batch: iter_time=1.898e-04, forward_time=0.150, loss_ctc=1.051, loss=1.051, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.077e-04, train_time=1.213 +[ALab02] 2023-06-10 19:34:20,576 (trainer:721) INFO: 34epoch:train:6062-6612batch: iter_time=1.880e-04, forward_time=0.152, loss_ctc=1.026, loss=1.026, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.075e-04, train_time=1.226 +[ALab02] 2023-06-10 19:37:10,813 (trainer:721) INFO: 34epoch:train:6613-7163batch: iter_time=2.076e-04, forward_time=0.153, loss_ctc=0.996, loss=0.996, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.072e-04, train_time=1.237 +[ALab02] 2023-06-10 19:39:59,017 (trainer:721) INFO: 34epoch:train:7164-7714batch: iter_time=2.110e-04, forward_time=0.150, loss_ctc=1.009, loss=1.009, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.070e-04, train_time=1.221 +[ALab02] 2023-06-10 19:42:14,311 (trainer:721) INFO: 34epoch:train:7715-8265batch: iter_time=2.113e-04, forward_time=0.111, loss_ctc=1.038, loss=1.038, backward_time=0.093, optim_step_time=0.036, optim0_lr0=3.068e-04, train_time=0.983 +[ALab02] 2023-06-10 19:45:01,037 (trainer:721) INFO: 34epoch:train:8266-8816batch: iter_time=2.111e-04, forward_time=0.150, loss_ctc=1.005, loss=1.005, backward_time=0.100, optim_step_time=0.036, optim0_lr0=3.065e-04, train_time=1.209 +[ALab02] 2023-06-10 19:47:49,412 (trainer:721) INFO: 34epoch:train:8817-9367batch: iter_time=2.077e-04, forward_time=0.150, loss_ctc=1.014, loss=1.014, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.063e-04, train_time=1.223 +[ALab02] 2023-06-10 19:50:39,190 (trainer:721) INFO: 34epoch:train:9368-9918batch: iter_time=2.190e-04, forward_time=0.150, loss_ctc=1.020, loss=1.020, backward_time=0.103, optim_step_time=0.038, optim0_lr0=3.061e-04, train_time=1.232 +[ALab02] 2023-06-10 19:53:27,566 (trainer:721) INFO: 34epoch:train:9919-10469batch: iter_time=1.980e-04, forward_time=0.150, loss_ctc=0.988, loss=0.988, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.059e-04, train_time=1.222 +[ALab02] 2023-06-10 19:56:21,095 (trainer:721) INFO: 34epoch:train:10470-11020batch: iter_time=2.078e-04, forward_time=0.157, loss_ctc=1.035, loss=1.035, backward_time=0.103, optim_step_time=0.036, optim0_lr0=3.056e-04, train_time=1.260 +[ALab02] 2023-06-10 19:57:51,912 (trainer:338) INFO: 34epoch results: [train] iter_time=2.325e-04, forward_time=0.146, loss_ctc=1.039, loss=1.039, backward_time=0.099, optim_step_time=0.035, optim0_lr0=3.078e-04, train_time=1.194, time=54 minutes and 52.03 seconds, total_count=374986, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.165, cer_ctc=0.055, cer=0.055, loss=4.165, time=57.15 seconds, total_count=14994, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.6 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 19:57:54,312 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 19:57:54,313 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/23epoch.pth +[ALab02] 2023-06-10 19:57:54,313 (trainer:272) INFO: 35/70epoch started. Estimated time to finish: 1 day, 9 hours and 11 minutes +[ALab02] 2023-06-10 20:00:41,895 (trainer:721) INFO: 35epoch:train:1-551batch: iter_time=8.988e-04, forward_time=0.150, loss_ctc=0.974, loss=0.974, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.054e-04, train_time=1.217 +[ALab02] 2023-06-10 20:03:28,443 (trainer:721) INFO: 35epoch:train:552-1102batch: iter_time=3.028e-04, forward_time=0.149, loss_ctc=0.965, loss=0.965, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.052e-04, train_time=1.210 +[ALab02] 2023-06-10 20:06:18,371 (trainer:721) INFO: 35epoch:train:1103-1653batch: iter_time=2.705e-04, forward_time=0.152, loss_ctc=0.978, loss=0.978, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.050e-04, train_time=1.232 +[ALab02] 2023-06-10 20:09:05,207 (trainer:721) INFO: 35epoch:train:1654-2204batch: iter_time=2.295e-04, forward_time=0.149, loss_ctc=1.036, loss=1.036, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.047e-04, train_time=1.211 +[ALab02] 2023-06-10 20:11:56,047 (trainer:721) INFO: 35epoch:train:2205-2755batch: iter_time=2.038e-04, forward_time=0.154, loss_ctc=0.996, loss=0.996, backward_time=0.101, optim_step_time=0.034, optim0_lr0=3.045e-04, train_time=1.240 +[ALab02] 2023-06-10 20:14:44,299 (trainer:721) INFO: 35epoch:train:2756-3306batch: iter_time=1.879e-04, forward_time=0.152, loss_ctc=0.979, loss=0.979, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.043e-04, train_time=1.222 +[ALab02] 2023-06-10 20:16:59,352 (trainer:721) INFO: 35epoch:train:3307-3857batch: iter_time=2.030e-04, forward_time=0.112, loss_ctc=0.957, loss=0.957, backward_time=0.092, optim_step_time=0.036, optim0_lr0=3.041e-04, train_time=0.980 +[ALab02] 2023-06-10 20:19:46,805 (trainer:721) INFO: 35epoch:train:3858-4408batch: iter_time=1.986e-04, forward_time=0.149, loss_ctc=0.985, loss=0.985, backward_time=0.100, optim_step_time=0.034, optim0_lr0=3.039e-04, train_time=1.216 +[ALab02] 2023-06-10 20:22:34,498 (trainer:721) INFO: 35epoch:train:4409-4959batch: iter_time=1.998e-04, forward_time=0.151, loss_ctc=0.963, loss=0.963, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.036e-04, train_time=1.217 +[ALab02] 2023-06-10 20:25:24,727 (trainer:721) INFO: 35epoch:train:4960-5510batch: iter_time=2.037e-04, forward_time=0.154, loss_ctc=0.990, loss=0.990, backward_time=0.100, optim_step_time=0.035, optim0_lr0=3.034e-04, train_time=1.237 +[ALab02] 2023-06-10 20:28:11,968 (trainer:721) INFO: 35epoch:train:5511-6061batch: iter_time=2.045e-04, forward_time=0.150, loss_ctc=0.938, loss=0.938, backward_time=0.101, optim_step_time=0.035, optim0_lr0=3.032e-04, train_time=1.214 +[ALab02] 2023-06-10 20:31:01,837 (trainer:721) INFO: 35epoch:train:6062-6612batch: iter_time=2.038e-04, forward_time=0.152, loss_ctc=0.949, loss=0.949, backward_time=0.102, optim_step_time=0.035, optim0_lr0=3.030e-04, train_time=1.233 +[ALab02] 2023-06-10 20:33:49,355 (trainer:721) INFO: 35epoch:train:6613-7163batch: iter_time=1.925e-04, forward_time=0.151, loss_ctc=0.976, loss=0.976, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.028e-04, train_time=1.217 +[ALab02] 2023-06-10 20:36:36,463 (trainer:721) INFO: 35epoch:train:7164-7714batch: iter_time=1.573e-04, forward_time=0.151, loss_ctc=0.994, loss=0.994, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.025e-04, train_time=1.212 +[ALab02] 2023-06-10 20:39:21,051 (trainer:721) INFO: 35epoch:train:7715-8265batch: iter_time=1.526e-04, forward_time=0.148, loss_ctc=0.986, loss=0.986, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.023e-04, train_time=1.195 +[ALab02] 2023-06-10 20:42:05,853 (trainer:721) INFO: 35epoch:train:8266-8816batch: iter_time=1.550e-04, forward_time=0.148, loss_ctc=0.971, loss=0.971, backward_time=0.098, optim_step_time=0.032, optim0_lr0=3.021e-04, train_time=1.195 +[ALab02] 2023-06-10 20:44:51,236 (trainer:721) INFO: 35epoch:train:8817-9367batch: iter_time=1.588e-04, forward_time=0.149, loss_ctc=0.959, loss=0.959, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.019e-04, train_time=1.202 +[ALab02] 2023-06-10 20:47:38,260 (trainer:721) INFO: 35epoch:train:9368-9918batch: iter_time=1.546e-04, forward_time=0.149, loss_ctc=0.987, loss=0.987, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.017e-04, train_time=1.212 +[ALab02] 2023-06-10 20:50:25,020 (trainer:721) INFO: 35epoch:train:9919-10469batch: iter_time=1.486e-04, forward_time=0.150, loss_ctc=0.963, loss=0.963, backward_time=0.098, optim_step_time=0.033, optim0_lr0=3.015e-04, train_time=1.209 +[ALab02] 2023-06-10 20:52:33,727 (trainer:721) INFO: 35epoch:train:10470-11020batch: iter_time=1.616e-04, forward_time=0.106, loss_ctc=0.988, loss=0.988, backward_time=0.086, optim_step_time=0.034, optim0_lr0=3.012e-04, train_time=0.935 +[ALab02] 2023-06-10 20:54:02,727 (trainer:338) INFO: 35epoch results: [train] iter_time=2.293e-04, forward_time=0.146, loss_ctc=0.977, loss=0.977, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.033e-04, train_time=1.190, time=54 minutes and 42.32 seconds, total_count=386015, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.147, cer_ctc=0.055, cer=0.055, loss=4.147, time=56.32 seconds, total_count=15435, gpu_max_cached_mem_GB=10.861, [att_plot] time=29.77 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 20:54:05,247 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-10 20:54:05,248 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/28epoch.pth +[ALab02] 2023-06-10 20:54:05,248 (trainer:272) INFO: 36/70epoch started. Estimated time to finish: 1 day, 8 hours and 16 minutes +[ALab02] 2023-06-10 20:56:48,979 (trainer:721) INFO: 36epoch:train:1-551batch: iter_time=6.105e-04, forward_time=0.146, loss_ctc=0.938, loss=0.938, backward_time=0.097, optim_step_time=0.033, optim0_lr0=3.010e-04, train_time=1.188 +[ALab02] 2023-06-10 20:59:33,715 (trainer:721) INFO: 36epoch:train:552-1102batch: iter_time=1.855e-04, forward_time=0.147, loss_ctc=0.934, loss=0.934, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.008e-04, train_time=1.196 +[ALab02] 2023-06-10 21:02:19,540 (trainer:721) INFO: 36epoch:train:1103-1653batch: iter_time=1.785e-04, forward_time=0.149, loss_ctc=1.002, loss=1.002, backward_time=0.098, optim_step_time=0.034, optim0_lr0=3.006e-04, train_time=1.204 +[ALab02] 2023-06-10 21:05:04,891 (trainer:721) INFO: 36epoch:train:1654-2204batch: iter_time=1.874e-04, forward_time=0.147, loss_ctc=0.959, loss=0.959, backward_time=0.099, optim_step_time=0.034, optim0_lr0=3.004e-04, train_time=1.200 +[ALab02] 2023-06-10 21:07:53,147 (trainer:721) INFO: 36epoch:train:2205-2755batch: iter_time=1.754e-04, forward_time=0.151, loss_ctc=0.975, loss=0.975, backward_time=0.099, optim_step_time=0.033, optim0_lr0=3.002e-04, train_time=1.221 +[ALab02] 2023-06-10 21:10:41,095 (trainer:721) INFO: 36epoch:train:2756-3306batch: iter_time=1.884e-04, forward_time=0.150, loss_ctc=0.982, loss=0.982, backward_time=0.100, optim_step_time=0.033, optim0_lr0=3.000e-04, train_time=1.221 +[ALab02] 2023-06-10 21:13:28,129 (trainer:721) INFO: 36epoch:train:3307-3857batch: iter_time=1.831e-04, forward_time=0.149, loss_ctc=0.948, loss=0.948, backward_time=0.100, optim_step_time=0.033, optim0_lr0=2.997e-04, train_time=1.211 +[ALab02] 2023-06-10 21:16:16,158 (trainer:721) INFO: 36epoch:train:3858-4408batch: iter_time=1.825e-04, forward_time=0.150, loss_ctc=0.992, loss=0.992, backward_time=0.101, optim_step_time=0.033, optim0_lr0=2.995e-04, train_time=1.220 +[ALab02] 2023-06-10 21:19:04,601 (trainer:721) INFO: 36epoch:train:4409-4959batch: iter_time=1.862e-04, forward_time=0.152, loss_ctc=0.982, loss=0.982, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.993e-04, train_time=1.222 +[ALab02] 2023-06-10 21:21:53,351 (trainer:721) INFO: 36epoch:train:4960-5510batch: iter_time=1.925e-04, forward_time=0.152, loss_ctc=1.044, loss=1.044, backward_time=0.100, optim_step_time=0.033, optim0_lr0=2.991e-04, train_time=1.227 +[ALab02] 2023-06-10 21:24:41,641 (trainer:721) INFO: 36epoch:train:5511-6061batch: iter_time=2.015e-04, forward_time=0.150, loss_ctc=1.018, loss=1.018, backward_time=0.101, optim_step_time=0.036, optim0_lr0=2.989e-04, train_time=1.221 +[ALab02] 2023-06-10 21:26:52,028 (trainer:721) INFO: 36epoch:train:6062-6612batch: iter_time=1.864e-04, forward_time=0.108, loss_ctc=1.026, loss=1.026, backward_time=0.087, optim_step_time=0.034, optim0_lr0=2.987e-04, train_time=0.946 +[ALab02] 2023-06-10 21:29:42,284 (trainer:721) INFO: 36epoch:train:6613-7163batch: iter_time=1.934e-04, forward_time=0.153, loss_ctc=1.001, loss=1.001, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.985e-04, train_time=1.237 +[ALab02] 2023-06-10 21:32:27,535 (trainer:721) INFO: 36epoch:train:7164-7714batch: iter_time=1.921e-04, forward_time=0.147, loss_ctc=1.011, loss=1.011, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.983e-04, train_time=1.199 +[ALab02] 2023-06-10 21:35:15,965 (trainer:721) INFO: 36epoch:train:7715-8265batch: iter_time=1.984e-04, forward_time=0.151, loss_ctc=1.033, loss=1.033, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.981e-04, train_time=1.222 +[ALab02] 2023-06-10 21:38:03,898 (trainer:721) INFO: 36epoch:train:8266-8816batch: iter_time=1.781e-04, forward_time=0.150, loss_ctc=1.020, loss=1.020, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.978e-04, train_time=1.219 +[ALab02] 2023-06-10 21:40:51,020 (trainer:721) INFO: 36epoch:train:8817-9367batch: iter_time=1.833e-04, forward_time=0.150, loss_ctc=1.071, loss=1.071, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.976e-04, train_time=1.214 +[ALab02] 2023-06-10 21:43:35,397 (trainer:721) INFO: 36epoch:train:9368-9918batch: iter_time=1.948e-04, forward_time=0.148, loss_ctc=1.018, loss=1.018, backward_time=0.098, optim_step_time=0.034, optim0_lr0=2.974e-04, train_time=1.195 +[ALab02] 2023-06-10 21:46:23,821 (trainer:721) INFO: 36epoch:train:9919-10469batch: iter_time=1.930e-04, forward_time=0.152, loss_ctc=0.997, loss=0.997, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.972e-04, train_time=1.220 +[ALab02] 2023-06-10 21:49:08,929 (trainer:721) INFO: 36epoch:train:10470-11020batch: iter_time=1.852e-04, forward_time=0.146, loss_ctc=1.030, loss=1.030, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.970e-04, train_time=1.198 +[ALab02] 2023-06-10 21:50:38,841 (trainer:338) INFO: 36epoch results: [train] iter_time=2.088e-04, forward_time=0.147, loss_ctc=0.999, loss=0.999, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.990e-04, train_time=1.199, time=55 minutes and 6.67 seconds, total_count=397044, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.156, cer_ctc=0.056, cer=0.056, loss=4.156, time=56.61 seconds, total_count=15876, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.3 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 21:50:41,436 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 21:50:41,438 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/27epoch.pth +[ALab02] 2023-06-10 21:50:41,438 (trainer:272) INFO: 37/70epoch started. Estimated time to finish: 1 day, 7 hours and 22 minutes +[ALab02] 2023-06-10 21:53:28,470 (trainer:721) INFO: 37epoch:train:1-551batch: iter_time=0.001, forward_time=0.149, loss_ctc=0.921, loss=0.921, backward_time=0.099, optim_step_time=0.033, optim0_lr0=2.968e-04, train_time=1.215 +[ALab02] 2023-06-10 21:56:16,858 (trainer:721) INFO: 37epoch:train:552-1102batch: iter_time=2.773e-04, forward_time=0.152, loss_ctc=0.948, loss=0.948, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.966e-04, train_time=1.219 +[ALab02] 2023-06-10 21:59:02,550 (trainer:721) INFO: 37epoch:train:1103-1653batch: iter_time=1.719e-04, forward_time=0.148, loss_ctc=0.977, loss=0.977, backward_time=0.099, optim_step_time=0.033, optim0_lr0=2.964e-04, train_time=1.204 +[ALab02] 2023-06-10 22:01:15,050 (trainer:721) INFO: 37epoch:train:1654-2204batch: iter_time=1.998e-04, forward_time=0.109, loss_ctc=0.989, loss=0.989, backward_time=0.089, optim_step_time=0.035, optim0_lr0=2.962e-04, train_time=0.961 +[ALab02] 2023-06-10 22:04:01,545 (trainer:721) INFO: 37epoch:train:2205-2755batch: iter_time=1.817e-04, forward_time=0.151, loss_ctc=1.014, loss=1.014, backward_time=0.098, optim_step_time=0.034, optim0_lr0=2.960e-04, train_time=1.210 +[ALab02] 2023-06-10 22:06:46,355 (trainer:721) INFO: 37epoch:train:2756-3306batch: iter_time=1.901e-04, forward_time=0.148, loss_ctc=1.035, loss=1.035, backward_time=0.099, optim_step_time=0.035, optim0_lr0=2.958e-04, train_time=1.197 +[ALab02] 2023-06-10 22:09:33,030 (trainer:721) INFO: 37epoch:train:3307-3857batch: iter_time=1.866e-04, forward_time=0.149, loss_ctc=0.977, loss=0.977, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.956e-04, train_time=1.208 +[ALab02] 2023-06-10 22:12:22,800 (trainer:721) INFO: 37epoch:train:3858-4408batch: iter_time=2.096e-04, forward_time=0.152, loss_ctc=0.964, loss=0.964, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.954e-04, train_time=1.232 +[ALab02] 2023-06-10 22:15:11,797 (trainer:721) INFO: 37epoch:train:4409-4959batch: iter_time=2.029e-04, forward_time=0.152, loss_ctc=0.976, loss=0.976, backward_time=0.100, optim_step_time=0.036, optim0_lr0=2.952e-04, train_time=1.225 +[ALab02] 2023-06-10 22:17:58,763 (trainer:721) INFO: 37epoch:train:4960-5510batch: iter_time=1.941e-04, forward_time=0.149, loss_ctc=0.964, loss=0.964, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.950e-04, train_time=1.213 +[ALab02] 2023-06-10 22:20:48,157 (trainer:721) INFO: 37epoch:train:5511-6061batch: iter_time=1.804e-04, forward_time=0.153, loss_ctc=0.996, loss=0.996, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.948e-04, train_time=1.230 +[ALab02] 2023-06-10 22:23:39,339 (trainer:721) INFO: 37epoch:train:6062-6612batch: iter_time=2.074e-04, forward_time=0.153, loss_ctc=0.952, loss=0.952, backward_time=0.103, optim_step_time=0.036, optim0_lr0=2.946e-04, train_time=1.241 +[ALab02] 2023-06-10 22:26:26,687 (trainer:721) INFO: 37epoch:train:6613-7163batch: iter_time=1.913e-04, forward_time=0.150, loss_ctc=0.952, loss=0.952, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.944e-04, train_time=1.214 +[ALab02] 2023-06-10 22:29:14,413 (trainer:721) INFO: 37epoch:train:7164-7714batch: iter_time=1.924e-04, forward_time=0.149, loss_ctc=0.983, loss=0.983, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.942e-04, train_time=1.217 +[ALab02] 2023-06-10 22:32:04,481 (trainer:721) INFO: 37epoch:train:7715-8265batch: iter_time=2.005e-04, forward_time=0.152, loss_ctc=0.941, loss=0.941, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.940e-04, train_time=1.235 +[ALab02] 2023-06-10 22:34:54,138 (trainer:721) INFO: 37epoch:train:8266-8816batch: iter_time=1.928e-04, forward_time=0.154, loss_ctc=0.920, loss=0.920, backward_time=0.100, optim_step_time=0.034, optim0_lr0=2.938e-04, train_time=1.232 +[ALab02] 2023-06-10 22:37:08,844 (trainer:721) INFO: 37epoch:train:8817-9367batch: iter_time=1.926e-04, forward_time=0.110, loss_ctc=0.947, loss=0.947, backward_time=0.092, optim_step_time=0.036, optim0_lr0=2.936e-04, train_time=0.973 +[ALab02] 2023-06-10 22:40:21,443 (trainer:721) INFO: 37epoch:train:9368-9918batch: iter_time=2.952e-04, forward_time=0.170, loss_ctc=0.976, loss=0.976, backward_time=0.123, optim_step_time=0.048, optim0_lr0=2.934e-04, train_time=1.401 +[ALab02] 2023-06-10 22:43:11,126 (trainer:721) INFO: 37epoch:train:9919-10469batch: iter_time=1.881e-04, forward_time=0.154, loss_ctc=0.966, loss=0.966, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.932e-04, train_time=1.232 +[ALab02] 2023-06-10 22:45:57,053 (trainer:721) INFO: 37epoch:train:10470-11020batch: iter_time=1.983e-04, forward_time=0.150, loss_ctc=0.972, loss=0.972, backward_time=0.099, optim_step_time=0.035, optim0_lr0=2.930e-04, train_time=1.203 +[ALab02] 2023-06-10 22:47:26,681 (trainer:338) INFO: 37epoch results: [train] iter_time=2.517e-04, forward_time=0.148, loss_ctc=0.968, loss=0.968, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.949e-04, train_time=1.203, time=55 minutes and 18.57 seconds, total_count=408073, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.154, cer_ctc=0.055, cer=0.055, loss=4.154, time=56.05 seconds, total_count=16317, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.63 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 22:47:29,259 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 22:47:29,261 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/29epoch.pth +[ALab02] 2023-06-10 22:47:29,261 (trainer:272) INFO: 38/70epoch started. Estimated time to finish: 1 day, 6 hours and 28 minutes +[ALab02] 2023-06-10 22:50:20,866 (trainer:721) INFO: 38epoch:train:1-551batch: iter_time=9.166e-04, forward_time=0.154, loss_ctc=0.936, loss=0.936, backward_time=0.102, optim_step_time=0.035, optim0_lr0=2.928e-04, train_time=1.246 +[ALab02] 2023-06-10 22:53:08,359 (trainer:721) INFO: 38epoch:train:552-1102batch: iter_time=2.022e-04, forward_time=0.149, loss_ctc=0.959, loss=0.959, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.926e-04, train_time=1.216 +[ALab02] 2023-06-10 22:55:58,058 (trainer:721) INFO: 38epoch:train:1103-1653batch: iter_time=2.065e-04, forward_time=0.154, loss_ctc=0.919, loss=0.919, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.924e-04, train_time=1.233 +[ALab02] 2023-06-10 22:58:44,511 (trainer:721) INFO: 38epoch:train:1654-2204batch: iter_time=2.187e-04, forward_time=0.149, loss_ctc=0.916, loss=0.916, backward_time=0.100, optim_step_time=0.036, optim0_lr0=2.922e-04, train_time=1.207 +[ALab02] 2023-06-10 23:01:35,725 (trainer:721) INFO: 38epoch:train:2205-2755batch: iter_time=1.978e-04, forward_time=0.154, loss_ctc=0.938, loss=0.938, backward_time=0.102, optim_step_time=0.035, optim0_lr0=2.920e-04, train_time=1.243 +[ALab02] 2023-06-10 23:04:22,046 (trainer:721) INFO: 38epoch:train:2756-3306batch: iter_time=1.973e-04, forward_time=0.149, loss_ctc=0.933, loss=0.933, backward_time=0.099, optim_step_time=0.034, optim0_lr0=2.918e-04, train_time=1.207 +[ALab02] 2023-06-10 23:07:11,084 (trainer:721) INFO: 38epoch:train:3307-3857batch: iter_time=1.876e-04, forward_time=0.152, loss_ctc=0.902, loss=0.902, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.916e-04, train_time=1.228 +[ALab02] 2023-06-10 23:09:57,753 (trainer:721) INFO: 38epoch:train:3858-4408batch: iter_time=1.973e-04, forward_time=0.148, loss_ctc=0.913, loss=0.913, backward_time=0.101, optim_step_time=0.034, optim0_lr0=2.914e-04, train_time=1.209 +[ALab02] 2023-06-10 23:12:10,209 (trainer:721) INFO: 38epoch:train:4409-4959batch: iter_time=1.993e-04, forward_time=0.109, loss_ctc=0.912, loss=0.912, backward_time=0.089, optim_step_time=0.036, optim0_lr0=2.912e-04, train_time=0.960 +[ALab02] 2023-06-10 23:14:56,887 (trainer:721) INFO: 38epoch:train:4960-5510batch: iter_time=2.027e-04, forward_time=0.149, loss_ctc=0.932, loss=0.932, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.910e-04, train_time=1.211 +[ALab02] 2023-06-10 23:17:43,342 (trainer:721) INFO: 38epoch:train:5511-6061batch: iter_time=1.965e-04, forward_time=0.148, loss_ctc=0.969, loss=0.969, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.908e-04, train_time=1.206 +[ALab02] 2023-06-10 23:20:32,192 (trainer:721) INFO: 38epoch:train:6062-6612batch: iter_time=2.028e-04, forward_time=0.152, loss_ctc=0.910, loss=0.910, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.906e-04, train_time=1.225 +[ALab02] 2023-06-10 23:23:21,404 (trainer:721) INFO: 38epoch:train:6613-7163batch: iter_time=2.178e-04, forward_time=0.154, loss_ctc=0.904, loss=0.904, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.904e-04, train_time=1.229 +[ALab02] 2023-06-10 23:26:12,036 (trainer:721) INFO: 38epoch:train:7164-7714batch: iter_time=2.119e-04, forward_time=0.153, loss_ctc=0.902, loss=0.902, backward_time=0.102, optim_step_time=0.035, optim0_lr0=2.902e-04, train_time=1.238 +[ALab02] 2023-06-10 23:29:01,500 (trainer:721) INFO: 38epoch:train:7715-8265batch: iter_time=1.986e-04, forward_time=0.153, loss_ctc=0.935, loss=0.935, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.900e-04, train_time=1.230 +[ALab02] 2023-06-10 23:31:48,836 (trainer:721) INFO: 38epoch:train:8266-8816batch: iter_time=2.011e-04, forward_time=0.150, loss_ctc=0.940, loss=0.940, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.899e-04, train_time=1.214 +[ALab02] 2023-06-10 23:34:37,466 (trainer:721) INFO: 38epoch:train:8817-9367batch: iter_time=1.886e-04, forward_time=0.150, loss_ctc=0.923, loss=0.923, backward_time=0.101, optim_step_time=0.036, optim0_lr0=2.897e-04, train_time=1.226 +[ALab02] 2023-06-10 23:37:26,440 (trainer:721) INFO: 38epoch:train:9368-9918batch: iter_time=2.071e-04, forward_time=0.152, loss_ctc=0.903, loss=0.903, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.895e-04, train_time=1.225 +[ALab02] 2023-06-10 23:40:13,671 (trainer:721) INFO: 38epoch:train:9919-10469batch: iter_time=2.084e-04, forward_time=0.151, loss_ctc=0.928, loss=0.928, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.893e-04, train_time=1.215 +[ALab02] 2023-06-10 23:43:03,615 (trainer:721) INFO: 38epoch:train:10470-11020batch: iter_time=2.052e-04, forward_time=0.152, loss_ctc=0.892, loss=0.892, backward_time=0.102, optim_step_time=0.036, optim0_lr0=2.891e-04, train_time=1.232 +[ALab02] 2023-06-10 23:44:34,065 (trainer:338) INFO: 38epoch results: [train] iter_time=2.381e-04, forward_time=0.149, loss_ctc=0.923, loss=0.923, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.909e-04, train_time=1.210, time=55 minutes and 37.59 seconds, total_count=419102, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.175, cer_ctc=0.055, cer=0.055, loss=4.175, time=56.35 seconds, total_count=16758, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.86 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-10 23:44:36,449 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-10 23:44:36,450 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/26epoch.pth +[ALab02] 2023-06-10 23:44:36,450 (trainer:272) INFO: 39/70epoch started. Estimated time to finish: 1 day, 5 hours and 34 minutes +[ALab02] 2023-06-10 23:46:54,297 (trainer:721) INFO: 39epoch:train:1-551batch: iter_time=0.001, forward_time=0.113, loss_ctc=0.887, loss=0.887, backward_time=0.093, optim_step_time=0.038, optim0_lr0=2.889e-04, train_time=0.998 +[ALab02] 2023-06-10 23:49:44,320 (trainer:721) INFO: 39epoch:train:552-1102batch: iter_time=3.395e-04, forward_time=0.153, loss_ctc=0.930, loss=0.930, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.887e-04, train_time=1.237 +[ALab02] 2023-06-10 23:52:34,808 (trainer:721) INFO: 39epoch:train:1103-1653batch: iter_time=2.572e-04, forward_time=0.153, loss_ctc=0.891, loss=0.891, backward_time=0.102, optim_step_time=0.037, optim0_lr0=2.885e-04, train_time=1.236 +[ALab02] 2023-06-10 23:55:25,393 (trainer:721) INFO: 39epoch:train:1654-2204batch: iter_time=2.095e-04, forward_time=0.155, loss_ctc=0.924, loss=0.924, backward_time=0.100, optim_step_time=0.035, optim0_lr0=2.883e-04, train_time=1.238 +[ALab02] 2023-06-10 23:58:15,049 (trainer:721) INFO: 39epoch:train:2205-2755batch: iter_time=2.100e-04, forward_time=0.151, loss_ctc=0.863, loss=0.863, backward_time=0.102, optim_step_time=0.036, optim0_lr0=2.881e-04, train_time=1.233 +[ALab02] 2023-06-11 00:01:33,099 (trainer:721) INFO: 39epoch:train:2756-3306batch: iter_time=3.095e-04, forward_time=0.174, loss_ctc=0.869, loss=0.869, backward_time=0.130, optim_step_time=0.053, optim0_lr0=2.880e-04, train_time=1.434 +[ALab02] 2023-06-11 00:04:59,176 (trainer:721) INFO: 39epoch:train:3307-3857batch: iter_time=3.538e-04, forward_time=0.180, loss_ctc=0.943, loss=0.943, backward_time=0.139, optim_step_time=0.061, optim0_lr0=2.878e-04, train_time=1.497 +[ALab02] 2023-06-11 00:08:29,723 (trainer:721) INFO: 39epoch:train:3858-4408batch: iter_time=4.112e-04, forward_time=0.185, loss_ctc=0.925, loss=0.925, backward_time=0.141, optim_step_time=0.065, optim0_lr0=2.876e-04, train_time=1.527 +[ALab02] 2023-06-11 00:11:49,711 (trainer:721) INFO: 39epoch:train:4409-4959batch: iter_time=3.866e-04, forward_time=0.177, loss_ctc=0.903, loss=0.903, backward_time=0.130, optim_step_time=0.057, optim0_lr0=2.874e-04, train_time=1.452 +[ALab02] 2023-06-11 00:15:15,391 (trainer:721) INFO: 39epoch:train:4960-5510batch: iter_time=3.513e-04, forward_time=0.182, loss_ctc=0.941, loss=0.941, backward_time=0.134, optim_step_time=0.061, optim0_lr0=2.872e-04, train_time=1.492 +[ALab02] 2023-06-11 00:18:40,194 (trainer:721) INFO: 39epoch:train:5511-6061batch: iter_time=3.636e-04, forward_time=0.181, loss_ctc=0.919, loss=0.919, backward_time=0.133, optim_step_time=0.063, optim0_lr0=2.870e-04, train_time=1.486 +[ALab02] 2023-06-11 00:21:35,124 (trainer:721) INFO: 39epoch:train:6062-6612batch: iter_time=3.480e-04, forward_time=0.141, loss_ctc=0.929, loss=0.929, backward_time=0.129, optim_step_time=0.059, optim0_lr0=2.868e-04, train_time=1.270 +[ALab02] 2023-06-11 00:24:57,147 (trainer:721) INFO: 39epoch:train:6613-7163batch: iter_time=3.488e-04, forward_time=0.178, loss_ctc=0.941, loss=0.941, backward_time=0.131, optim_step_time=0.061, optim0_lr0=2.866e-04, train_time=1.468 +[ALab02] 2023-06-11 00:28:19,395 (trainer:721) INFO: 39epoch:train:7164-7714batch: iter_time=3.473e-04, forward_time=0.177, loss_ctc=0.929, loss=0.929, backward_time=0.133, optim_step_time=0.060, optim0_lr0=2.865e-04, train_time=1.468 +[ALab02] 2023-06-11 00:31:43,812 (trainer:721) INFO: 39epoch:train:7715-8265batch: iter_time=3.547e-04, forward_time=0.180, loss_ctc=0.914, loss=0.914, backward_time=0.134, optim_step_time=0.060, optim0_lr0=2.863e-04, train_time=1.482 +[ALab02] 2023-06-11 00:35:08,343 (trainer:721) INFO: 39epoch:train:8266-8816batch: iter_time=3.527e-04, forward_time=0.180, loss_ctc=0.878, loss=0.878, backward_time=0.134, optim_step_time=0.061, optim0_lr0=2.861e-04, train_time=1.484 +[ALab02] 2023-06-11 00:38:32,532 (trainer:721) INFO: 39epoch:train:8817-9367batch: iter_time=3.351e-04, forward_time=0.178, loss_ctc=0.913, loss=0.913, backward_time=0.136, optim_step_time=0.062, optim0_lr0=2.859e-04, train_time=1.482 +[ALab02] 2023-06-11 00:41:57,823 (trainer:721) INFO: 39epoch:train:9368-9918batch: iter_time=3.603e-04, forward_time=0.180, loss_ctc=0.856, loss=0.856, backward_time=0.136, optim_step_time=0.059, optim0_lr0=2.857e-04, train_time=1.490 +[ALab02] 2023-06-11 00:45:20,286 (trainer:721) INFO: 39epoch:train:9919-10469batch: iter_time=3.525e-04, forward_time=0.177, loss_ctc=0.904, loss=0.904, backward_time=0.133, optim_step_time=0.059, optim0_lr0=2.855e-04, train_time=1.469 +[ALab02] 2023-06-11 00:48:41,738 (trainer:721) INFO: 39epoch:train:10470-11020batch: iter_time=3.761e-04, forward_time=0.178, loss_ctc=0.846, loss=0.846, backward_time=0.131, optim_step_time=0.059, optim0_lr0=2.854e-04, train_time=1.462 +[ALab02] 2023-06-11 00:50:38,298 (trainer:338) INFO: 39epoch results: [train] iter_time=3.691e-04, forward_time=0.169, loss_ctc=0.905, loss=0.905, backward_time=0.125, optim_step_time=0.054, optim0_lr0=2.871e-04, train_time=1.395, time=1 hour, 4 minutes and 9.34 seconds, total_count=430131, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.089, cer_ctc=0.053, cer=0.053, loss=4.089, time=59.51 seconds, total_count=17199, gpu_max_cached_mem_GB=10.861, [att_plot] time=53 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 00:50:42,852 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-11 00:50:42,860 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/25epoch.pth +[ALab02] 2023-06-11 00:50:42,861 (trainer:272) INFO: 40/70epoch started. Estimated time to finish: 1 day, 4 hours and 47 minutes +[ALab02] 2023-06-11 00:54:04,410 (trainer:721) INFO: 40epoch:train:1-551batch: iter_time=0.002, forward_time=0.175, loss_ctc=0.849, loss=0.849, backward_time=0.131, optim_step_time=0.060, optim0_lr0=2.852e-04, train_time=1.464 +[ALab02] 2023-06-11 00:56:57,428 (trainer:721) INFO: 40epoch:train:552-1102batch: iter_time=3.752e-04, forward_time=0.142, loss_ctc=0.870, loss=0.870, backward_time=0.126, optim_step_time=0.058, optim0_lr0=2.850e-04, train_time=1.257 +[ALab02] 2023-06-11 01:00:16,875 (trainer:721) INFO: 40epoch:train:1103-1653batch: iter_time=3.742e-04, forward_time=0.175, loss_ctc=0.859, loss=0.859, backward_time=0.133, optim_step_time=0.057, optim0_lr0=2.848e-04, train_time=1.445 +[ALab02] 2023-06-11 01:03:35,750 (trainer:721) INFO: 40epoch:train:1654-2204batch: iter_time=3.559e-04, forward_time=0.175, loss_ctc=0.885, loss=0.885, backward_time=0.130, optim_step_time=0.055, optim0_lr0=2.846e-04, train_time=1.444 +[ALab02] 2023-06-11 01:06:53,998 (trainer:721) INFO: 40epoch:train:2205-2755batch: iter_time=3.394e-04, forward_time=0.176, loss_ctc=0.852, loss=0.852, backward_time=0.127, optim_step_time=0.053, optim0_lr0=2.844e-04, train_time=1.439 +[ALab02] 2023-06-11 01:10:06,389 (trainer:721) INFO: 40epoch:train:2756-3306batch: iter_time=3.548e-04, forward_time=0.168, loss_ctc=0.898, loss=0.898, backward_time=0.125, optim_step_time=0.054, optim0_lr0=2.843e-04, train_time=1.396 +[ALab02] 2023-06-11 01:13:15,697 (trainer:721) INFO: 40epoch:train:3307-3857batch: iter_time=3.116e-04, forward_time=0.166, loss_ctc=0.878, loss=0.878, backward_time=0.121, optim_step_time=0.051, optim0_lr0=2.841e-04, train_time=1.375 +[ALab02] 2023-06-11 01:16:17,208 (trainer:721) INFO: 40epoch:train:3858-4408batch: iter_time=2.801e-04, forward_time=0.162, loss_ctc=0.883, loss=0.883, backward_time=0.112, optim_step_time=0.044, optim0_lr0=2.839e-04, train_time=1.317 +[ALab02] 2023-06-11 01:19:13,764 (trainer:721) INFO: 40epoch:train:4409-4959batch: iter_time=2.474e-04, forward_time=0.157, loss_ctc=0.901, loss=0.901, backward_time=0.109, optim_step_time=0.040, optim0_lr0=2.837e-04, train_time=1.282 +[ALab02] 2023-06-11 01:22:08,270 (trainer:721) INFO: 40epoch:train:4960-5510batch: iter_time=2.532e-04, forward_time=0.157, loss_ctc=0.871, loss=0.871, backward_time=0.105, optim_step_time=0.038, optim0_lr0=2.835e-04, train_time=1.267 +[ALab02] 2023-06-11 01:25:03,354 (trainer:721) INFO: 40epoch:train:5511-6061batch: iter_time=2.341e-04, forward_time=0.157, loss_ctc=0.880, loss=0.880, backward_time=0.106, optim_step_time=0.040, optim0_lr0=2.834e-04, train_time=1.271 +[ALab02] 2023-06-11 01:28:14,394 (trainer:721) INFO: 40epoch:train:6062-6612batch: iter_time=3.378e-04, forward_time=0.170, loss_ctc=0.880, loss=0.880, backward_time=0.121, optim_step_time=0.051, optim0_lr0=2.832e-04, train_time=1.386 +[ALab02] 2023-06-11 01:31:28,429 (trainer:721) INFO: 40epoch:train:6613-7163batch: iter_time=3.209e-04, forward_time=0.171, loss_ctc=0.909, loss=0.909, backward_time=0.124, optim_step_time=0.055, optim0_lr0=2.830e-04, train_time=1.410 +[ALab02] 2023-06-11 01:34:17,926 (trainer:721) INFO: 40epoch:train:7164-7714batch: iter_time=3.352e-04, forward_time=0.137, loss_ctc=0.912, loss=0.912, backward_time=0.124, optim_step_time=0.055, optim0_lr0=2.828e-04, train_time=1.230 +[ALab02] 2023-06-11 01:37:31,533 (trainer:721) INFO: 40epoch:train:7715-8265batch: iter_time=3.224e-04, forward_time=0.170, loss_ctc=0.875, loss=0.875, backward_time=0.125, optim_step_time=0.056, optim0_lr0=2.827e-04, train_time=1.404 +[ALab02] 2023-06-11 01:40:44,652 (trainer:721) INFO: 40epoch:train:8266-8816batch: iter_time=3.671e-04, forward_time=0.171, loss_ctc=0.932, loss=0.932, backward_time=0.124, optim_step_time=0.055, optim0_lr0=2.825e-04, train_time=1.401 +[ALab02] 2023-06-11 01:43:57,078 (trainer:721) INFO: 40epoch:train:8817-9367batch: iter_time=3.138e-04, forward_time=0.167, loss_ctc=0.899, loss=0.899, backward_time=0.126, optim_step_time=0.054, optim0_lr0=2.823e-04, train_time=1.397 +[ALab02] 2023-06-11 01:47:08,878 (trainer:721) INFO: 40epoch:train:9368-9918batch: iter_time=3.121e-04, forward_time=0.169, loss_ctc=0.881, loss=0.881, backward_time=0.125, optim_step_time=0.051, optim0_lr0=2.821e-04, train_time=1.391 +[ALab02] 2023-06-11 01:50:28,645 (trainer:721) INFO: 40epoch:train:9919-10469batch: iter_time=3.742e-04, forward_time=0.175, loss_ctc=0.880, loss=0.880, backward_time=0.132, optim_step_time=0.060, optim0_lr0=2.819e-04, train_time=1.450 +[ALab02] 2023-06-11 01:53:36,374 (trainer:721) INFO: 40epoch:train:10470-11020batch: iter_time=3.153e-04, forward_time=0.166, loss_ctc=0.926, loss=0.926, backward_time=0.118, optim_step_time=0.050, optim0_lr0=2.818e-04, train_time=1.362 +[ALab02] 2023-06-11 01:55:21,732 (trainer:338) INFO: 40epoch results: [train] iter_time=3.930e-04, forward_time=0.165, loss_ctc=0.886, loss=0.886, backward_time=0.122, optim_step_time=0.052, optim0_lr0=2.835e-04, train_time=1.369, time=1 hour, 2 minutes and 57.3 seconds, total_count=441160, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.255, cer_ctc=0.055, cer=0.055, loss=4.255, time=58.34 seconds, total_count=17640, gpu_max_cached_mem_GB=10.861, [att_plot] time=43.23 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 01:55:25,360 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 01:55:25,363 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/31epoch.pth +[ALab02] 2023-06-11 01:55:25,364 (trainer:272) INFO: 41/70epoch started. Estimated time to finish: 1 day, 3 hours and 58 minutes +[ALab02] 2023-06-11 01:58:28,677 (trainer:721) INFO: 41epoch:train:1-551batch: iter_time=0.002, forward_time=0.162, loss_ctc=0.842, loss=0.842, backward_time=0.112, optim_step_time=0.046, optim0_lr0=2.816e-04, train_time=1.332 +[ALab02] 2023-06-11 02:01:22,503 (trainer:721) INFO: 41epoch:train:552-1102batch: iter_time=2.580e-04, forward_time=0.155, loss_ctc=0.887, loss=0.887, backward_time=0.106, optim_step_time=0.040, optim0_lr0=2.814e-04, train_time=1.261 +[ALab02] 2023-06-11 02:04:14,654 (trainer:721) INFO: 41epoch:train:1103-1653batch: iter_time=2.329e-04, forward_time=0.154, loss_ctc=0.868, loss=0.868, backward_time=0.104, optim_step_time=0.038, optim0_lr0=2.812e-04, train_time=1.249 +[ALab02] 2023-06-11 02:07:04,825 (trainer:721) INFO: 41epoch:train:1654-2204batch: iter_time=2.069e-04, forward_time=0.155, loss_ctc=0.872, loss=0.872, backward_time=0.101, optim_step_time=0.035, optim0_lr0=2.811e-04, train_time=1.235 +[ALab02] 2023-06-11 02:09:51,793 (trainer:721) INFO: 41epoch:train:2205-2755batch: iter_time=3.358e-04, forward_time=0.134, loss_ctc=0.872, loss=0.872, backward_time=0.123, optim_step_time=0.055, optim0_lr0=2.809e-04, train_time=1.211 +[ALab02] 2023-06-11 02:13:05,863 (trainer:721) INFO: 41epoch:train:2756-3306batch: iter_time=3.331e-04, forward_time=0.171, loss_ctc=0.878, loss=0.878, backward_time=0.124, optim_step_time=0.057, optim0_lr0=2.807e-04, train_time=1.409 +[ALab02] 2023-06-11 02:16:22,233 (trainer:721) INFO: 41epoch:train:3307-3857batch: iter_time=3.470e-04, forward_time=0.173, loss_ctc=0.880, loss=0.880, backward_time=0.126, optim_step_time=0.057, optim0_lr0=2.805e-04, train_time=1.424 +[ALab02] 2023-06-11 02:19:35,493 (trainer:721) INFO: 41epoch:train:3858-4408batch: iter_time=3.323e-04, forward_time=0.169, loss_ctc=0.861, loss=0.861, backward_time=0.126, optim_step_time=0.056, optim0_lr0=2.804e-04, train_time=1.404 +[ALab02] 2023-06-11 02:22:50,396 (trainer:721) INFO: 41epoch:train:4409-4959batch: iter_time=3.268e-04, forward_time=0.171, loss_ctc=0.903, loss=0.903, backward_time=0.126, optim_step_time=0.056, optim0_lr0=2.802e-04, train_time=1.416 +[ALab02] 2023-06-11 02:26:09,371 (trainer:721) INFO: 41epoch:train:4960-5510batch: iter_time=3.724e-04, forward_time=0.177, loss_ctc=0.893, loss=0.893, backward_time=0.128, optim_step_time=0.058, optim0_lr0=2.800e-04, train_time=1.442 +[ALab02] 2023-06-11 02:29:23,535 (trainer:721) INFO: 41epoch:train:5511-6061batch: iter_time=3.364e-04, forward_time=0.172, loss_ctc=0.883, loss=0.883, backward_time=0.124, optim_step_time=0.056, optim0_lr0=2.798e-04, train_time=1.410 +[ALab02] 2023-06-11 02:32:42,585 (trainer:721) INFO: 41epoch:train:6062-6612batch: iter_time=3.392e-04, forward_time=0.177, loss_ctc=0.857, loss=0.857, backward_time=0.127, optim_step_time=0.059, optim0_lr0=2.797e-04, train_time=1.444 +[ALab02] 2023-06-11 02:35:59,495 (trainer:721) INFO: 41epoch:train:6613-7163batch: iter_time=3.473e-04, forward_time=0.174, loss_ctc=0.872, loss=0.872, backward_time=0.127, optim_step_time=0.056, optim0_lr0=2.795e-04, train_time=1.429 +[ALab02] 2023-06-11 02:39:14,160 (trainer:721) INFO: 41epoch:train:7164-7714batch: iter_time=3.335e-04, forward_time=0.172, loss_ctc=0.853, loss=0.853, backward_time=0.125, optim_step_time=0.057, optim0_lr0=2.793e-04, train_time=1.414 +[ALab02] 2023-06-11 02:42:27,362 (trainer:721) INFO: 41epoch:train:7715-8265batch: iter_time=3.452e-04, forward_time=0.170, loss_ctc=0.884, loss=0.884, backward_time=0.125, optim_step_time=0.054, optim0_lr0=2.792e-04, train_time=1.401 +[ALab02] 2023-06-11 02:45:09,565 (trainer:721) INFO: 41epoch:train:8266-8816batch: iter_time=3.802e-04, forward_time=0.119, loss_ctc=0.875, loss=0.875, backward_time=0.132, optim_step_time=0.063, optim0_lr0=2.790e-04, train_time=1.177 +[ALab02] 2023-06-11 02:48:23,535 (trainer:721) INFO: 41epoch:train:8817-9367batch: iter_time=5.264e-04, forward_time=0.134, loss_ctc=0.915, loss=0.915, backward_time=0.172, optim_step_time=0.084, optim0_lr0=2.788e-04, train_time=1.411 +[ALab02] 2023-06-11 02:50:39,908 (trainer:721) INFO: 41epoch:train:9368-9918batch: iter_time=3.517e-04, forward_time=0.095, loss_ctc=0.875, loss=0.875, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.786e-04, train_time=0.990 +[ALab02] 2023-06-11 02:52:50,266 (trainer:721) INFO: 41epoch:train:9919-10469batch: iter_time=3.390e-04, forward_time=0.088, loss_ctc=0.907, loss=0.907, backward_time=0.115, optim_step_time=0.057, optim0_lr0=2.785e-04, train_time=0.946 +[ALab02] 2023-06-11 02:55:03,885 (trainer:721) INFO: 41epoch:train:10470-11020batch: iter_time=3.402e-04, forward_time=0.092, loss_ctc=0.878, loss=0.878, backward_time=0.116, optim_step_time=0.058, optim0_lr0=2.783e-04, train_time=0.969 +[ALab02] 2023-06-11 02:56:24,621 (trainer:338) INFO: 41epoch results: [train] iter_time=4.240e-04, forward_time=0.151, loss_ctc=0.878, loss=0.878, backward_time=0.123, optim_step_time=0.055, optim0_lr0=2.799e-04, train_time=1.298, time=59 minutes and 41.36 seconds, total_count=452189, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.280, cer_ctc=0.055, cer=0.055, loss=4.280, time=32.3 seconds, total_count=18081, gpu_max_cached_mem_GB=10.861, [att_plot] time=45.59 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 02:56:28,335 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 02:56:28,337 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/32epoch.pth +[ALab02] 2023-06-11 02:56:28,337 (trainer:272) INFO: 42/70epoch started. Estimated time to finish: 1 day, 3 hours and 6 minutes +[ALab02] 2023-06-11 02:58:42,137 (trainer:721) INFO: 42epoch:train:1-551batch: iter_time=0.002, forward_time=0.091, loss_ctc=0.916, loss=0.916, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.781e-04, train_time=0.971 +[ALab02] 2023-06-11 03:00:52,708 (trainer:721) INFO: 42epoch:train:552-1102batch: iter_time=3.531e-04, forward_time=0.088, loss_ctc=0.880, loss=0.880, backward_time=0.116, optim_step_time=0.053, optim0_lr0=2.780e-04, train_time=0.948 +[ALab02] 2023-06-11 03:03:01,518 (trainer:721) INFO: 42epoch:train:1103-1653batch: iter_time=3.508e-04, forward_time=0.088, loss_ctc=0.855, loss=0.855, backward_time=0.113, optim_step_time=0.055, optim0_lr0=2.778e-04, train_time=0.934 +[ALab02] 2023-06-11 03:05:10,889 (trainer:721) INFO: 42epoch:train:1654-2204batch: iter_time=3.262e-04, forward_time=0.087, loss_ctc=0.853, loss=0.853, backward_time=0.115, optim_step_time=0.053, optim0_lr0=2.776e-04, train_time=0.939 +[ALab02] 2023-06-11 03:07:19,432 (trainer:721) INFO: 42epoch:train:2205-2755batch: iter_time=3.199e-04, forward_time=0.088, loss_ctc=0.867, loss=0.867, backward_time=0.113, optim_step_time=0.053, optim0_lr0=2.775e-04, train_time=0.934 +[ALab02] 2023-06-11 03:09:29,080 (trainer:721) INFO: 42epoch:train:2756-3306batch: iter_time=3.424e-04, forward_time=0.088, loss_ctc=0.914, loss=0.914, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.773e-04, train_time=0.940 +[ALab02] 2023-06-11 03:11:42,620 (trainer:721) INFO: 42epoch:train:3307-3857batch: iter_time=3.288e-04, forward_time=0.091, loss_ctc=0.863, loss=0.863, backward_time=0.118, optim_step_time=0.055, optim0_lr0=2.771e-04, train_time=0.969 +[ALab02] 2023-06-11 03:13:53,862 (trainer:721) INFO: 42epoch:train:3858-4408batch: iter_time=3.284e-04, forward_time=0.089, loss_ctc=0.889, loss=0.889, backward_time=0.116, optim_step_time=0.054, optim0_lr0=2.770e-04, train_time=0.952 +[ALab02] 2023-06-11 03:16:01,796 (trainer:721) INFO: 42epoch:train:4409-4959batch: iter_time=3.191e-04, forward_time=0.088, loss_ctc=0.886, loss=0.886, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.768e-04, train_time=0.929 +[ALab02] 2023-06-11 03:18:07,905 (trainer:721) INFO: 42epoch:train:4960-5510batch: iter_time=3.129e-04, forward_time=0.085, loss_ctc=0.897, loss=0.897, backward_time=0.111, optim_step_time=0.053, optim0_lr0=2.766e-04, train_time=0.915 +[ALab02] 2023-06-11 03:20:22,523 (trainer:721) INFO: 42epoch:train:5511-6061batch: iter_time=3.375e-04, forward_time=0.091, loss_ctc=0.854, loss=0.854, backward_time=0.120, optim_step_time=0.055, optim0_lr0=2.765e-04, train_time=0.976 +[ALab02] 2023-06-11 03:22:37,590 (trainer:721) INFO: 42epoch:train:6062-6612batch: iter_time=3.574e-04, forward_time=0.093, loss_ctc=0.886, loss=0.886, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.763e-04, train_time=0.980 +[ALab02] 2023-06-11 03:24:49,137 (trainer:721) INFO: 42epoch:train:6613-7163batch: iter_time=3.210e-04, forward_time=0.092, loss_ctc=0.870, loss=0.870, backward_time=0.112, optim_step_time=0.057, optim0_lr0=2.761e-04, train_time=0.956 +[ALab02] 2023-06-11 03:26:59,621 (trainer:721) INFO: 42epoch:train:7164-7714batch: iter_time=3.113e-04, forward_time=0.089, loss_ctc=0.814, loss=0.814, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.760e-04, train_time=0.946 +[ALab02] 2023-06-11 03:29:13,084 (trainer:721) INFO: 42epoch:train:7715-8265batch: iter_time=3.231e-04, forward_time=0.091, loss_ctc=0.842, loss=0.842, backward_time=0.116, optim_step_time=0.057, optim0_lr0=2.758e-04, train_time=0.968 +[ALab02] 2023-06-11 03:31:36,243 (trainer:721) INFO: 42epoch:train:8266-8816batch: iter_time=3.796e-04, forward_time=0.099, loss_ctc=0.812, loss=0.812, backward_time=0.125, optim_step_time=0.062, optim0_lr0=2.756e-04, train_time=1.038 +[ALab02] 2023-06-11 03:33:49,402 (trainer:721) INFO: 42epoch:train:8817-9367batch: iter_time=3.508e-04, forward_time=0.091, loss_ctc=0.830, loss=0.830, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.755e-04, train_time=0.967 +[ALab02] 2023-06-11 03:36:03,213 (trainer:721) INFO: 42epoch:train:9368-9918batch: iter_time=3.310e-04, forward_time=0.092, loss_ctc=0.853, loss=0.853, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.753e-04, train_time=0.970 +[ALab02] 2023-06-11 03:38:18,248 (trainer:721) INFO: 42epoch:train:9919-10469batch: iter_time=3.240e-04, forward_time=0.093, loss_ctc=0.824, loss=0.824, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.751e-04, train_time=0.980 +[ALab02] 2023-06-11 03:40:33,075 (trainer:721) INFO: 42epoch:train:10470-11020batch: iter_time=3.511e-04, forward_time=0.093, loss_ctc=0.830, loss=0.830, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.750e-04, train_time=0.978 +[ALab02] 2023-06-11 03:41:56,479 (trainer:338) INFO: 42epoch results: [train] iter_time=4.171e-04, forward_time=0.090, loss_ctc=0.861, loss=0.861, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.765e-04, train_time=0.960, time=44 minutes and 7.28 seconds, total_count=463218, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.127, cer_ctc=0.054, cer=0.054, loss=4.127, time=34.48 seconds, total_count=18522, gpu_max_cached_mem_GB=10.861, [att_plot] time=46.38 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 03:42:00,401 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 03:42:00,403 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/30epoch.pth +[ALab02] 2023-06-11 03:42:00,404 (trainer:272) INFO: 43/70epoch started. Estimated time to finish: 1 day, 2 hours and 3 minutes +[ALab02] 2023-06-11 03:44:12,523 (trainer:721) INFO: 43epoch:train:1-551batch: iter_time=0.002, forward_time=0.088, loss_ctc=0.839, loss=0.839, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.748e-04, train_time=0.959 +[ALab02] 2023-06-11 03:46:22,982 (trainer:721) INFO: 43epoch:train:552-1102batch: iter_time=3.480e-04, forward_time=0.089, loss_ctc=0.794, loss=0.794, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.746e-04, train_time=0.947 +[ALab02] 2023-06-11 03:48:29,924 (trainer:721) INFO: 43epoch:train:1103-1653batch: iter_time=3.285e-04, forward_time=0.086, loss_ctc=0.810, loss=0.810, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.745e-04, train_time=0.921 +[ALab02] 2023-06-11 03:50:41,177 (trainer:721) INFO: 43epoch:train:1654-2204batch: iter_time=3.243e-04, forward_time=0.090, loss_ctc=0.793, loss=0.793, backward_time=0.115, optim_step_time=0.054, optim0_lr0=2.743e-04, train_time=0.952 +[ALab02] 2023-06-11 03:52:51,367 (trainer:721) INFO: 43epoch:train:2205-2755batch: iter_time=3.243e-04, forward_time=0.089, loss_ctc=0.808, loss=0.808, backward_time=0.114, optim_step_time=0.056, optim0_lr0=2.742e-04, train_time=0.946 +[ALab02] 2023-06-11 03:55:00,659 (trainer:721) INFO: 43epoch:train:2756-3306batch: iter_time=3.196e-04, forward_time=0.088, loss_ctc=0.822, loss=0.822, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.740e-04, train_time=0.937 +[ALab02] 2023-06-11 03:57:17,147 (trainer:721) INFO: 43epoch:train:3307-3857batch: iter_time=3.402e-04, forward_time=0.092, loss_ctc=0.816, loss=0.816, backward_time=0.122, optim_step_time=0.057, optim0_lr0=2.738e-04, train_time=0.990 +[ALab02] 2023-06-11 03:59:32,820 (trainer:721) INFO: 43epoch:train:3858-4408batch: iter_time=3.441e-04, forward_time=0.092, loss_ctc=0.807, loss=0.807, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.737e-04, train_time=0.985 +[ALab02] 2023-06-11 04:01:46,841 (trainer:721) INFO: 43epoch:train:4409-4959batch: iter_time=3.321e-04, forward_time=0.092, loss_ctc=0.824, loss=0.824, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.735e-04, train_time=0.973 +[ALab02] 2023-06-11 04:04:00,894 (trainer:721) INFO: 43epoch:train:4960-5510batch: iter_time=3.228e-04, forward_time=0.092, loss_ctc=0.853, loss=0.853, backward_time=0.118, optim_step_time=0.055, optim0_lr0=2.734e-04, train_time=0.972 +[ALab02] 2023-06-11 04:06:13,367 (trainer:721) INFO: 43epoch:train:5511-6061batch: iter_time=3.319e-04, forward_time=0.090, loss_ctc=0.818, loss=0.818, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.732e-04, train_time=0.960 +[ALab02] 2023-06-11 04:08:25,114 (trainer:721) INFO: 43epoch:train:6062-6612batch: iter_time=3.288e-04, forward_time=0.088, loss_ctc=0.848, loss=0.848, backward_time=0.118, optim_step_time=0.054, optim0_lr0=2.730e-04, train_time=0.956 +[ALab02] 2023-06-11 04:10:43,570 (trainer:721) INFO: 43epoch:train:6613-7163batch: iter_time=3.797e-04, forward_time=0.096, loss_ctc=0.833, loss=0.833, backward_time=0.120, optim_step_time=0.059, optim0_lr0=2.729e-04, train_time=1.005 +[ALab02] 2023-06-11 04:12:55,774 (trainer:721) INFO: 43epoch:train:7164-7714batch: iter_time=3.284e-04, forward_time=0.091, loss_ctc=0.834, loss=0.834, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.727e-04, train_time=0.959 +[ALab02] 2023-06-11 04:15:10,387 (trainer:721) INFO: 43epoch:train:7715-8265batch: iter_time=3.268e-04, forward_time=0.093, loss_ctc=0.843, loss=0.843, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.725e-04, train_time=0.976 +[ALab02] 2023-06-11 04:17:24,977 (trainer:721) INFO: 43epoch:train:8266-8816batch: iter_time=3.323e-04, forward_time=0.092, loss_ctc=0.846, loss=0.846, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.724e-04, train_time=0.977 +[ALab02] 2023-06-11 04:19:40,010 (trainer:721) INFO: 43epoch:train:8817-9367batch: iter_time=3.302e-04, forward_time=0.093, loss_ctc=0.838, loss=0.838, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.722e-04, train_time=0.978 +[ALab02] 2023-06-11 04:21:59,523 (trainer:721) INFO: 43epoch:train:9368-9918batch: iter_time=3.482e-04, forward_time=0.096, loss_ctc=0.840, loss=0.840, backward_time=0.121, optim_step_time=0.060, optim0_lr0=2.721e-04, train_time=1.014 +[ALab02] 2023-06-11 04:24:14,964 (trainer:721) INFO: 43epoch:train:9919-10469batch: iter_time=3.548e-04, forward_time=0.095, loss_ctc=0.874, loss=0.874, backward_time=0.116, optim_step_time=0.058, optim0_lr0=2.719e-04, train_time=0.983 +[ALab02] 2023-06-11 04:26:31,094 (trainer:721) INFO: 43epoch:train:10470-11020batch: iter_time=3.407e-04, forward_time=0.094, loss_ctc=0.863, loss=0.863, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.718e-04, train_time=0.988 +[ALab02] 2023-06-11 04:27:50,463 (trainer:338) INFO: 43epoch results: [train] iter_time=4.065e-04, forward_time=0.091, loss_ctc=0.830, loss=0.830, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.733e-04, train_time=0.969, time=44 minutes and 33.38 seconds, total_count=474247, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.235, cer_ctc=0.053, cer=0.053, loss=4.235, time=33.88 seconds, total_count=18963, gpu_max_cached_mem_GB=10.861, [att_plot] time=42.8 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 04:27:53,737 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-11 04:27:53,739 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/33epoch.pth +[ALab02] 2023-06-11 04:27:53,739 (trainer:272) INFO: 44/70epoch started. Estimated time to finish: 1 day, 1 hour and 1 minute +[ALab02] 2023-06-11 04:30:08,579 (trainer:721) INFO: 44epoch:train:1-551batch: iter_time=0.002, forward_time=0.091, loss_ctc=0.830, loss=0.830, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.716e-04, train_time=0.978 +[ALab02] 2023-06-11 04:32:20,743 (trainer:721) INFO: 44epoch:train:552-1102batch: iter_time=5.280e-04, forward_time=0.090, loss_ctc=0.804, loss=0.804, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.714e-04, train_time=0.960 +[ALab02] 2023-06-11 04:34:34,351 (trainer:721) INFO: 44epoch:train:1103-1653batch: iter_time=4.062e-04, forward_time=0.091, loss_ctc=0.807, loss=0.807, backward_time=0.118, optim_step_time=0.055, optim0_lr0=2.713e-04, train_time=0.969 +[ALab02] 2023-06-11 04:36:50,323 (trainer:721) INFO: 44epoch:train:1654-2204batch: iter_time=3.375e-04, forward_time=0.095, loss_ctc=0.837, loss=0.837, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.711e-04, train_time=0.986 +[ALab02] 2023-06-11 04:39:02,098 (trainer:721) INFO: 44epoch:train:2205-2755batch: iter_time=3.338e-04, forward_time=0.090, loss_ctc=0.819, loss=0.819, backward_time=0.116, optim_step_time=0.054, optim0_lr0=2.710e-04, train_time=0.956 +[ALab02] 2023-06-11 04:41:18,519 (trainer:721) INFO: 44epoch:train:2756-3306batch: iter_time=3.401e-04, forward_time=0.095, loss_ctc=0.836, loss=0.836, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.708e-04, train_time=0.990 +[ALab02] 2023-06-11 04:43:36,370 (trainer:721) INFO: 44epoch:train:3307-3857batch: iter_time=3.659e-04, forward_time=0.095, loss_ctc=0.812, loss=0.812, backward_time=0.121, optim_step_time=0.059, optim0_lr0=2.707e-04, train_time=1.001 +[ALab02] 2023-06-11 04:45:51,154 (trainer:721) INFO: 44epoch:train:3858-4408batch: iter_time=3.450e-04, forward_time=0.093, loss_ctc=0.834, loss=0.834, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.705e-04, train_time=0.978 +[ALab02] 2023-06-11 04:48:02,668 (trainer:721) INFO: 44epoch:train:4409-4959batch: iter_time=3.253e-04, forward_time=0.089, loss_ctc=0.813, loss=0.813, backward_time=0.116, optim_step_time=0.055, optim0_lr0=2.703e-04, train_time=0.955 +[ALab02] 2023-06-11 04:50:17,109 (trainer:721) INFO: 44epoch:train:4960-5510batch: iter_time=3.286e-04, forward_time=0.091, loss_ctc=0.833, loss=0.833, backward_time=0.120, optim_step_time=0.055, optim0_lr0=2.702e-04, train_time=0.975 +[ALab02] 2023-06-11 04:52:31,034 (trainer:721) INFO: 44epoch:train:5511-6061batch: iter_time=3.342e-04, forward_time=0.092, loss_ctc=0.836, loss=0.836, backward_time=0.118, optim_step_time=0.054, optim0_lr0=2.700e-04, train_time=0.972 +[ALab02] 2023-06-11 04:54:43,970 (trainer:721) INFO: 44epoch:train:6062-6612batch: iter_time=3.488e-04, forward_time=0.091, loss_ctc=0.835, loss=0.835, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.699e-04, train_time=0.964 +[ALab02] 2023-06-11 04:57:01,298 (trainer:721) INFO: 44epoch:train:6613-7163batch: iter_time=3.542e-04, forward_time=0.095, loss_ctc=0.834, loss=0.834, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.697e-04, train_time=0.996 +[ALab02] 2023-06-11 04:59:17,321 (trainer:721) INFO: 44epoch:train:7164-7714batch: iter_time=3.397e-04, forward_time=0.094, loss_ctc=0.869, loss=0.869, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.696e-04, train_time=0.988 +[ALab02] 2023-06-11 05:01:36,310 (trainer:721) INFO: 44epoch:train:7715-8265batch: iter_time=3.396e-04, forward_time=0.097, loss_ctc=0.824, loss=0.824, backward_time=0.120, optim_step_time=0.061, optim0_lr0=2.694e-04, train_time=1.008 +[ALab02] 2023-06-11 05:03:57,097 (trainer:721) INFO: 44epoch:train:8266-8816batch: iter_time=3.954e-04, forward_time=0.097, loss_ctc=0.855, loss=0.855, backward_time=0.123, optim_step_time=0.061, optim0_lr0=2.693e-04, train_time=1.021 +[ALab02] 2023-06-11 05:06:17,891 (trainer:721) INFO: 44epoch:train:8817-9367batch: iter_time=3.615e-04, forward_time=0.097, loss_ctc=0.823, loss=0.823, backward_time=0.124, optim_step_time=0.058, optim0_lr0=2.691e-04, train_time=1.022 +[ALab02] 2023-06-11 05:08:32,358 (trainer:721) INFO: 44epoch:train:9368-9918batch: iter_time=3.288e-04, forward_time=0.092, loss_ctc=0.817, loss=0.817, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.690e-04, train_time=0.975 +[ALab02] 2023-06-11 05:11:04,138 (trainer:721) INFO: 44epoch:train:9919-10469batch: iter_time=3.718e-04, forward_time=0.104, loss_ctc=0.845, loss=0.845, backward_time=0.134, optim_step_time=0.064, optim0_lr0=2.688e-04, train_time=1.102 +[ALab02] 2023-06-11 05:13:20,045 (trainer:721) INFO: 44epoch:train:10470-11020batch: iter_time=3.673e-04, forward_time=0.092, loss_ctc=0.849, loss=0.849, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.686e-04, train_time=0.986 +[ALab02] 2023-06-11 05:14:46,944 (trainer:338) INFO: 44epoch results: [train] iter_time=4.399e-04, forward_time=0.093, loss_ctc=0.830, loss=0.830, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.701e-04, train_time=0.989, time=45 minutes and 29.21 seconds, total_count=485276, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.293, cer_ctc=0.054, cer=0.054, loss=4.293, time=32.42 seconds, total_count=19404, gpu_max_cached_mem_GB=10.861, [att_plot] time=51.57 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 05:14:50,915 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 05:14:50,917 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/36epoch.pth +[ALab02] 2023-06-11 05:14:50,918 (trainer:272) INFO: 45/70epoch started. Estimated time to finish: 1 day and 19.68 seconds +[ALab02] 2023-06-11 05:17:10,843 (trainer:721) INFO: 45epoch:train:1-551batch: iter_time=0.002, forward_time=0.096, loss_ctc=0.833, loss=0.833, backward_time=0.121, optim_step_time=0.059, optim0_lr0=2.685e-04, train_time=1.016 +[ALab02] 2023-06-11 05:19:29,617 (trainer:721) INFO: 45epoch:train:552-1102batch: iter_time=3.538e-04, forward_time=0.095, loss_ctc=0.825, loss=0.825, backward_time=0.122, optim_step_time=0.060, optim0_lr0=2.683e-04, train_time=1.006 +[ALab02] 2023-06-11 05:21:52,626 (trainer:721) INFO: 45epoch:train:1103-1653batch: iter_time=3.432e-04, forward_time=0.097, loss_ctc=0.837, loss=0.837, backward_time=0.127, optim_step_time=0.061, optim0_lr0=2.682e-04, train_time=1.038 +[ALab02] 2023-06-11 05:24:10,022 (trainer:721) INFO: 45epoch:train:1654-2204batch: iter_time=3.639e-04, forward_time=0.095, loss_ctc=0.842, loss=0.842, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.680e-04, train_time=0.997 +[ALab02] 2023-06-11 05:26:31,532 (trainer:721) INFO: 45epoch:train:2205-2755batch: iter_time=3.488e-04, forward_time=0.096, loss_ctc=0.840, loss=0.840, backward_time=0.126, optim_step_time=0.059, optim0_lr0=2.679e-04, train_time=1.028 +[ALab02] 2023-06-11 05:28:48,701 (trainer:721) INFO: 45epoch:train:2756-3306batch: iter_time=3.872e-04, forward_time=0.095, loss_ctc=0.833, loss=0.833, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.677e-04, train_time=0.996 +[ALab02] 2023-06-11 05:31:08,597 (trainer:721) INFO: 45epoch:train:3307-3857batch: iter_time=3.531e-04, forward_time=0.097, loss_ctc=0.848, loss=0.848, backward_time=0.121, optim_step_time=0.060, optim0_lr0=2.676e-04, train_time=1.015 +[ALab02] 2023-06-11 05:33:30,572 (trainer:721) INFO: 45epoch:train:3858-4408batch: iter_time=3.476e-04, forward_time=0.097, loss_ctc=0.834, loss=0.834, backward_time=0.125, optim_step_time=0.061, optim0_lr0=2.674e-04, train_time=1.030 +[ALab02] 2023-06-11 05:35:51,383 (trainer:721) INFO: 45epoch:train:4409-4959batch: iter_time=3.418e-04, forward_time=0.097, loss_ctc=0.834, loss=0.834, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.673e-04, train_time=1.022 +[ALab02] 2023-06-11 05:38:12,722 (trainer:721) INFO: 45epoch:train:4960-5510batch: iter_time=3.498e-04, forward_time=0.099, loss_ctc=0.834, loss=0.834, backward_time=0.121, optim_step_time=0.061, optim0_lr0=2.671e-04, train_time=1.026 +[ALab02] 2023-06-11 05:40:31,974 (trainer:721) INFO: 45epoch:train:5511-6061batch: iter_time=3.855e-04, forward_time=0.098, loss_ctc=0.839, loss=0.839, backward_time=0.119, optim_step_time=0.060, optim0_lr0=2.670e-04, train_time=1.011 +[ALab02] 2023-06-11 05:42:51,948 (trainer:721) INFO: 45epoch:train:6062-6612batch: iter_time=3.334e-04, forward_time=0.095, loss_ctc=0.803, loss=0.803, backward_time=0.124, optim_step_time=0.060, optim0_lr0=2.668e-04, train_time=1.015 +[ALab02] 2023-06-11 05:45:11,619 (trainer:721) INFO: 45epoch:train:6613-7163batch: iter_time=3.900e-04, forward_time=0.097, loss_ctc=0.807, loss=0.807, backward_time=0.121, optim_step_time=0.060, optim0_lr0=2.667e-04, train_time=1.015 +[ALab02] 2023-06-11 05:47:34,681 (trainer:721) INFO: 45epoch:train:7164-7714batch: iter_time=3.656e-04, forward_time=0.098, loss_ctc=0.818, loss=0.818, backward_time=0.126, optim_step_time=0.060, optim0_lr0=2.665e-04, train_time=1.038 +[ALab02] 2023-06-11 05:49:53,053 (trainer:721) INFO: 45epoch:train:7715-8265batch: iter_time=3.761e-04, forward_time=0.094, loss_ctc=0.809, loss=0.809, backward_time=0.122, optim_step_time=0.059, optim0_lr0=2.664e-04, train_time=1.003 +[ALab02] 2023-06-11 05:52:16,596 (trainer:721) INFO: 45epoch:train:8266-8816batch: iter_time=3.620e-04, forward_time=0.097, loss_ctc=0.833, loss=0.833, backward_time=0.128, optim_step_time=0.059, optim0_lr0=2.662e-04, train_time=1.042 +[ALab02] 2023-06-11 05:54:35,491 (trainer:721) INFO: 45epoch:train:8817-9367batch: iter_time=3.445e-04, forward_time=0.094, loss_ctc=0.821, loss=0.821, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.661e-04, train_time=1.008 +[ALab02] 2023-06-11 05:56:56,865 (trainer:721) INFO: 45epoch:train:9368-9918batch: iter_time=3.732e-04, forward_time=0.096, loss_ctc=0.785, loss=0.785, backward_time=0.126, optim_step_time=0.059, optim0_lr0=2.659e-04, train_time=1.026 +[ALab02] 2023-06-11 05:59:15,323 (trainer:721) INFO: 45epoch:train:9919-10469batch: iter_time=3.640e-04, forward_time=0.095, loss_ctc=0.826, loss=0.826, backward_time=0.121, optim_step_time=0.059, optim0_lr0=2.658e-04, train_time=1.004 +[ALab02] 2023-06-11 06:01:35,370 (trainer:721) INFO: 45epoch:train:10470-11020batch: iter_time=3.560e-04, forward_time=0.095, loss_ctc=0.854, loss=0.854, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.656e-04, train_time=1.016 +[ALab02] 2023-06-11 06:03:01,683 (trainer:338) INFO: 45epoch results: [train] iter_time=4.315e-04, forward_time=0.096, loss_ctc=0.828, loss=0.828, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.671e-04, train_time=1.017, time=46 minutes and 47.36 seconds, total_count=496305, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.271, cer_ctc=0.054, cer=0.054, loss=4.271, time=34.93 seconds, total_count=19845, gpu_max_cached_mem_GB=10.861, [att_plot] time=48.47 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 06:03:05,076 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 06:03:05,078 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/34epoch.pth +[ALab02] 2023-06-11 06:03:05,078 (trainer:272) INFO: 46/70epoch started. Estimated time to finish: 23 hours and 57.14 seconds +[ALab02] 2023-06-11 06:05:23,580 (trainer:721) INFO: 46epoch:train:1-551batch: iter_time=0.002, forward_time=0.093, loss_ctc=0.790, loss=0.790, backward_time=0.122, optim_step_time=0.057, optim0_lr0=2.655e-04, train_time=1.005 +[ALab02] 2023-06-11 06:07:40,752 (trainer:721) INFO: 46epoch:train:552-1102batch: iter_time=3.519e-04, forward_time=0.094, loss_ctc=0.803, loss=0.803, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.653e-04, train_time=0.995 +[ALab02] 2023-06-11 06:09:56,804 (trainer:721) INFO: 46epoch:train:1103-1653batch: iter_time=3.331e-04, forward_time=0.093, loss_ctc=0.791, loss=0.791, backward_time=0.119, optim_step_time=0.058, optim0_lr0=2.652e-04, train_time=0.987 +[ALab02] 2023-06-11 06:12:21,446 (trainer:721) INFO: 46epoch:train:1654-2204batch: iter_time=4.197e-04, forward_time=0.098, loss_ctc=0.790, loss=0.790, backward_time=0.129, optim_step_time=0.060, optim0_lr0=2.651e-04, train_time=1.049 +[ALab02] 2023-06-11 06:14:40,849 (trainer:721) INFO: 46epoch:train:2205-2755batch: iter_time=3.536e-04, forward_time=0.095, loss_ctc=0.814, loss=0.814, backward_time=0.122, optim_step_time=0.060, optim0_lr0=2.649e-04, train_time=1.012 +[ALab02] 2023-06-11 06:16:59,920 (trainer:721) INFO: 46epoch:train:2756-3306batch: iter_time=3.615e-04, forward_time=0.094, loss_ctc=0.839, loss=0.839, backward_time=0.124, optim_step_time=0.059, optim0_lr0=2.648e-04, train_time=1.009 +[ALab02] 2023-06-11 06:19:19,039 (trainer:721) INFO: 46epoch:train:3307-3857batch: iter_time=3.472e-04, forward_time=0.095, loss_ctc=0.824, loss=0.824, backward_time=0.122, optim_step_time=0.061, optim0_lr0=2.646e-04, train_time=1.009 +[ALab02] 2023-06-11 06:21:37,523 (trainer:721) INFO: 46epoch:train:3858-4408batch: iter_time=3.505e-04, forward_time=0.096, loss_ctc=0.816, loss=0.816, backward_time=0.120, optim_step_time=0.060, optim0_lr0=2.645e-04, train_time=1.004 +[ALab02] 2023-06-11 06:23:54,320 (trainer:721) INFO: 46epoch:train:4409-4959batch: iter_time=3.696e-04, forward_time=0.094, loss_ctc=0.823, loss=0.823, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.643e-04, train_time=0.994 +[ALab02] 2023-06-11 06:26:12,400 (trainer:721) INFO: 46epoch:train:4960-5510batch: iter_time=3.430e-04, forward_time=0.096, loss_ctc=0.818, loss=0.818, backward_time=0.120, optim_step_time=0.059, optim0_lr0=2.642e-04, train_time=1.001 +[ALab02] 2023-06-11 06:28:26,341 (trainer:721) INFO: 46epoch:train:5511-6061batch: iter_time=3.876e-04, forward_time=0.093, loss_ctc=0.842, loss=0.842, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.640e-04, train_time=0.972 +[ALab02] 2023-06-11 06:30:45,115 (trainer:721) INFO: 46epoch:train:6062-6612batch: iter_time=3.786e-04, forward_time=0.094, loss_ctc=0.839, loss=0.839, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.639e-04, train_time=1.007 +[ALab02] 2023-06-11 06:33:00,609 (trainer:721) INFO: 46epoch:train:6613-7163batch: iter_time=3.368e-04, forward_time=0.093, loss_ctc=0.840, loss=0.840, backward_time=0.118, optim_step_time=0.059, optim0_lr0=2.637e-04, train_time=0.984 +[ALab02] 2023-06-11 06:35:19,352 (trainer:721) INFO: 46epoch:train:7164-7714batch: iter_time=3.405e-04, forward_time=0.095, loss_ctc=0.828, loss=0.828, backward_time=0.123, optim_step_time=0.057, optim0_lr0=2.636e-04, train_time=1.006 +[ALab02] 2023-06-11 06:37:34,084 (trainer:721) INFO: 46epoch:train:7715-8265batch: iter_time=3.271e-04, forward_time=0.091, loss_ctc=0.847, loss=0.847, backward_time=0.120, optim_step_time=0.055, optim0_lr0=2.635e-04, train_time=0.977 +[ALab02] 2023-06-11 06:39:50,721 (trainer:721) INFO: 46epoch:train:8266-8816batch: iter_time=3.575e-04, forward_time=0.093, loss_ctc=0.809, loss=0.809, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.633e-04, train_time=0.992 +[ALab02] 2023-06-11 06:42:02,568 (trainer:721) INFO: 46epoch:train:8817-9367batch: iter_time=3.360e-04, forward_time=0.091, loss_ctc=0.802, loss=0.802, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.632e-04, train_time=0.957 +[ALab02] 2023-06-11 06:44:19,047 (trainer:721) INFO: 46epoch:train:9368-9918batch: iter_time=3.238e-04, forward_time=0.090, loss_ctc=0.843, loss=0.843, backward_time=0.124, optim_step_time=0.056, optim0_lr0=2.630e-04, train_time=0.991 +[ALab02] 2023-06-11 06:46:34,427 (trainer:721) INFO: 46epoch:train:9919-10469batch: iter_time=3.337e-04, forward_time=0.094, loss_ctc=0.838, loss=0.838, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.629e-04, train_time=0.982 +[ALab02] 2023-06-11 06:48:48,831 (trainer:721) INFO: 46epoch:train:10470-11020batch: iter_time=3.324e-04, forward_time=0.092, loss_ctc=0.823, loss=0.823, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.627e-04, train_time=0.975 +[ALab02] 2023-06-11 06:50:20,803 (trainer:338) INFO: 46epoch results: [train] iter_time=4.483e-04, forward_time=0.094, loss_ctc=0.821, loss=0.821, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.641e-04, train_time=0.995, time=45 minutes and 46.57 seconds, total_count=507334, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.264, cer_ctc=0.053, cer=0.053, loss=4.264, time=36.29 seconds, total_count=20286, gpu_max_cached_mem_GB=10.861, [att_plot] time=52.87 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 06:50:24,949 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 06:50:24,951 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/41epoch.pth +[ALab02] 2023-06-11 06:50:24,952 (trainer:272) INFO: 47/70epoch started. Estimated time to finish: 22 hours, 1 minute and 35.33 seconds +[ALab02] 2023-06-11 06:52:43,049 (trainer:721) INFO: 47epoch:train:1-551batch: iter_time=0.002, forward_time=0.094, loss_ctc=0.792, loss=0.792, backward_time=0.121, optim_step_time=0.055, optim0_lr0=2.626e-04, train_time=1.003 +[ALab02] 2023-06-11 06:54:56,407 (trainer:721) INFO: 47epoch:train:552-1102batch: iter_time=3.414e-04, forward_time=0.092, loss_ctc=0.813, loss=0.813, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.625e-04, train_time=0.968 +[ALab02] 2023-06-11 06:57:08,713 (trainer:721) INFO: 47epoch:train:1103-1653batch: iter_time=3.362e-04, forward_time=0.092, loss_ctc=0.798, loss=0.798, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.623e-04, train_time=0.960 +[ALab02] 2023-06-11 06:59:20,193 (trainer:721) INFO: 47epoch:train:1654-2204batch: iter_time=3.427e-04, forward_time=0.089, loss_ctc=0.800, loss=0.800, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.622e-04, train_time=0.954 +[ALab02] 2023-06-11 07:01:32,035 (trainer:721) INFO: 47epoch:train:2205-2755batch: iter_time=3.278e-04, forward_time=0.091, loss_ctc=0.805, loss=0.805, backward_time=0.115, optim_step_time=0.055, optim0_lr0=2.620e-04, train_time=0.957 +[ALab02] 2023-06-11 07:03:39,990 (trainer:721) INFO: 47epoch:train:2756-3306batch: iter_time=3.632e-04, forward_time=0.086, loss_ctc=0.802, loss=0.802, backward_time=0.113, optim_step_time=0.053, optim0_lr0=2.619e-04, train_time=0.928 +[ALab02] 2023-06-11 07:05:53,148 (trainer:721) INFO: 47epoch:train:3307-3857batch: iter_time=3.296e-04, forward_time=0.091, loss_ctc=0.845, loss=0.845, backward_time=0.117, optim_step_time=0.055, optim0_lr0=2.617e-04, train_time=0.966 +[ALab02] 2023-06-11 07:08:02,934 (trainer:721) INFO: 47epoch:train:3858-4408batch: iter_time=3.182e-04, forward_time=0.088, loss_ctc=0.786, loss=0.786, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.616e-04, train_time=0.942 +[ALab02] 2023-06-11 07:10:10,412 (trainer:721) INFO: 47epoch:train:4409-4959batch: iter_time=3.219e-04, forward_time=0.086, loss_ctc=0.807, loss=0.807, backward_time=0.113, optim_step_time=0.053, optim0_lr0=2.615e-04, train_time=0.924 +[ALab02] 2023-06-11 07:12:28,377 (trainer:721) INFO: 47epoch:train:4960-5510batch: iter_time=3.606e-04, forward_time=0.096, loss_ctc=0.818, loss=0.818, backward_time=0.120, optim_step_time=0.059, optim0_lr0=2.613e-04, train_time=1.002 +[ALab02] 2023-06-11 07:14:39,844 (trainer:721) INFO: 47epoch:train:5511-6061batch: iter_time=3.260e-04, forward_time=0.089, loss_ctc=0.819, loss=0.819, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.612e-04, train_time=0.954 +[ALab02] 2023-06-11 07:16:56,794 (trainer:721) INFO: 47epoch:train:6062-6612batch: iter_time=3.417e-04, forward_time=0.094, loss_ctc=0.828, loss=0.828, backward_time=0.120, optim_step_time=0.059, optim0_lr0=2.610e-04, train_time=0.994 +[ALab02] 2023-06-11 07:19:15,091 (trainer:721) INFO: 47epoch:train:6613-7163batch: iter_time=3.567e-04, forward_time=0.095, loss_ctc=0.835, loss=0.835, backward_time=0.121, optim_step_time=0.059, optim0_lr0=2.609e-04, train_time=1.005 +[ALab02] 2023-06-11 07:21:29,525 (trainer:721) INFO: 47epoch:train:7164-7714batch: iter_time=3.628e-04, forward_time=0.094, loss_ctc=0.823, loss=0.823, backward_time=0.117, optim_step_time=0.055, optim0_lr0=2.608e-04, train_time=0.975 +[ALab02] 2023-06-11 07:23:48,502 (trainer:721) INFO: 47epoch:train:7715-8265batch: iter_time=3.460e-04, forward_time=0.095, loss_ctc=0.779, loss=0.779, backward_time=0.122, optim_step_time=0.059, optim0_lr0=2.606e-04, train_time=1.008 +[ALab02] 2023-06-11 07:26:04,894 (trainer:721) INFO: 47epoch:train:8266-8816batch: iter_time=3.455e-04, forward_time=0.094, loss_ctc=0.824, loss=0.824, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.605e-04, train_time=0.989 +[ALab02] 2023-06-11 07:28:18,864 (trainer:721) INFO: 47epoch:train:8817-9367batch: iter_time=3.267e-04, forward_time=0.092, loss_ctc=0.817, loss=0.817, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.603e-04, train_time=0.972 +[ALab02] 2023-06-11 07:30:33,845 (trainer:721) INFO: 47epoch:train:9368-9918batch: iter_time=3.378e-04, forward_time=0.092, loss_ctc=0.806, loss=0.806, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.602e-04, train_time=0.980 +[ALab02] 2023-06-11 07:32:49,701 (trainer:721) INFO: 47epoch:train:9919-10469batch: iter_time=3.355e-04, forward_time=0.096, loss_ctc=0.808, loss=0.808, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.601e-04, train_time=0.985 +[ALab02] 2023-06-11 07:35:05,872 (trainer:721) INFO: 47epoch:train:10470-11020batch: iter_time=3.395e-04, forward_time=0.093, loss_ctc=0.835, loss=0.835, backward_time=0.119, optim_step_time=0.058, optim0_lr0=2.599e-04, train_time=0.988 +[ALab02] 2023-06-11 07:36:27,858 (trainer:338) INFO: 47epoch results: [train] iter_time=4.126e-04, forward_time=0.092, loss_ctc=0.812, loss=0.812, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.613e-04, train_time=0.973, time=44 minutes and 43.91 seconds, total_count=518363, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.280, cer_ctc=0.054, cer=0.054, loss=4.280, time=32.34 seconds, total_count=20727, gpu_max_cached_mem_GB=10.861, [att_plot] time=46.65 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 07:36:31,719 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 07:36:31,721 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/38epoch.pth +[ALab02] 2023-06-11 07:36:31,721 (trainer:272) INFO: 48/70epoch started. Estimated time to finish: 21 hours, 2 minutes and 8.48 seconds +[ALab02] 2023-06-11 07:38:47,309 (trainer:721) INFO: 48epoch:train:1-551batch: iter_time=0.002, forward_time=0.093, loss_ctc=0.800, loss=0.800, backward_time=0.117, optim_step_time=0.058, optim0_lr0=2.598e-04, train_time=0.985 +[ALab02] 2023-06-11 07:41:03,002 (trainer:721) INFO: 48epoch:train:552-1102batch: iter_time=3.508e-04, forward_time=0.092, loss_ctc=0.783, loss=0.783, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.597e-04, train_time=0.983 +[ALab02] 2023-06-11 07:43:15,150 (trainer:721) INFO: 48epoch:train:1103-1653batch: iter_time=3.272e-04, forward_time=0.090, loss_ctc=0.810, loss=0.810, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.595e-04, train_time=0.959 +[ALab02] 2023-06-11 07:45:29,854 (trainer:721) INFO: 48epoch:train:1654-2204batch: iter_time=3.639e-04, forward_time=0.090, loss_ctc=0.788, loss=0.788, backward_time=0.121, optim_step_time=0.055, optim0_lr0=2.594e-04, train_time=0.977 +[ALab02] 2023-06-11 07:47:44,045 (trainer:721) INFO: 48epoch:train:2205-2755batch: iter_time=3.416e-04, forward_time=0.093, loss_ctc=0.817, loss=0.817, backward_time=0.116, optim_step_time=0.059, optim0_lr0=2.592e-04, train_time=0.975 +[ALab02] 2023-06-11 07:49:59,179 (trainer:721) INFO: 48epoch:train:2756-3306batch: iter_time=3.278e-04, forward_time=0.092, loss_ctc=0.807, loss=0.807, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.591e-04, train_time=0.980 +[ALab02] 2023-06-11 07:52:16,476 (trainer:721) INFO: 48epoch:train:3307-3857batch: iter_time=3.332e-04, forward_time=0.093, loss_ctc=0.827, loss=0.827, backward_time=0.122, optim_step_time=0.056, optim0_lr0=2.590e-04, train_time=0.996 +[ALab02] 2023-06-11 07:54:31,615 (trainer:721) INFO: 48epoch:train:3858-4408batch: iter_time=3.351e-04, forward_time=0.093, loss_ctc=0.796, loss=0.796, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.588e-04, train_time=0.980 +[ALab02] 2023-06-11 07:56:47,244 (trainer:721) INFO: 48epoch:train:4409-4959batch: iter_time=3.461e-04, forward_time=0.092, loss_ctc=0.786, loss=0.786, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.587e-04, train_time=0.986 +[ALab02] 2023-06-11 07:59:03,466 (trainer:721) INFO: 48epoch:train:4960-5510batch: iter_time=3.348e-04, forward_time=0.091, loss_ctc=0.796, loss=0.796, backward_time=0.122, optim_step_time=0.057, optim0_lr0=2.586e-04, train_time=0.988 +[ALab02] 2023-06-11 08:01:18,910 (trainer:721) INFO: 48epoch:train:5511-6061batch: iter_time=3.268e-04, forward_time=0.093, loss_ctc=0.825, loss=0.825, backward_time=0.119, optim_step_time=0.056, optim0_lr0=2.584e-04, train_time=0.982 +[ALab02] 2023-06-11 08:03:32,950 (trainer:721) INFO: 48epoch:train:6062-6612batch: iter_time=3.283e-04, forward_time=0.092, loss_ctc=0.778, loss=0.778, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.583e-04, train_time=0.973 +[ALab02] 2023-06-11 08:05:55,097 (trainer:721) INFO: 48epoch:train:6613-7163batch: iter_time=3.364e-04, forward_time=0.097, loss_ctc=0.794, loss=0.794, backward_time=0.125, optim_step_time=0.062, optim0_lr0=2.581e-04, train_time=1.033 +[ALab02] 2023-06-11 08:08:08,839 (trainer:721) INFO: 48epoch:train:7164-7714batch: iter_time=3.364e-04, forward_time=0.091, loss_ctc=0.821, loss=0.821, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.580e-04, train_time=0.970 +[ALab02] 2023-06-11 08:10:23,479 (trainer:721) INFO: 48epoch:train:7715-8265batch: iter_time=3.591e-04, forward_time=0.093, loss_ctc=0.831, loss=0.831, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.579e-04, train_time=0.977 +[ALab02] 2023-06-11 08:12:40,828 (trainer:721) INFO: 48epoch:train:8266-8816batch: iter_time=3.299e-04, forward_time=0.092, loss_ctc=0.857, loss=0.857, backward_time=0.123, optim_step_time=0.057, optim0_lr0=2.577e-04, train_time=0.996 +[ALab02] 2023-06-11 08:14:56,505 (trainer:721) INFO: 48epoch:train:8817-9367batch: iter_time=3.350e-04, forward_time=0.092, loss_ctc=0.873, loss=0.873, backward_time=0.120, optim_step_time=0.055, optim0_lr0=2.576e-04, train_time=0.985 +[ALab02] 2023-06-11 08:17:12,110 (trainer:721) INFO: 48epoch:train:9368-9918batch: iter_time=3.364e-04, forward_time=0.092, loss_ctc=0.808, loss=0.808, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.575e-04, train_time=0.983 +[ALab02] 2023-06-11 08:19:30,304 (trainer:721) INFO: 48epoch:train:9919-10469batch: iter_time=3.348e-04, forward_time=0.093, loss_ctc=0.878, loss=0.878, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.573e-04, train_time=1.004 +[ALab02] 2023-06-11 08:21:49,254 (trainer:721) INFO: 48epoch:train:10470-11020batch: iter_time=3.424e-04, forward_time=0.096, loss_ctc=0.845, loss=0.845, backward_time=0.121, optim_step_time=0.060, optim0_lr0=2.572e-04, train_time=1.008 +[ALab02] 2023-06-11 08:23:10,570 (trainer:338) INFO: 48epoch results: [train] iter_time=4.005e-04, forward_time=0.093, loss_ctc=0.816, loss=0.816, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.585e-04, train_time=0.986, time=45 minutes and 20.13 seconds, total_count=529392, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.372, cer_ctc=0.055, cer=0.055, loss=4.372, time=33.15 seconds, total_count=21168, gpu_max_cached_mem_GB=10.861, [att_plot] time=45.56 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 08:23:14,039 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 08:23:14,040 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/37epoch.pth +[ALab02] 2023-06-11 08:23:14,041 (trainer:272) INFO: 49/70epoch started. Estimated time to finish: 20 hours, 3 minutes and 31.25 seconds +[ALab02] 2023-06-11 08:25:29,807 (trainer:721) INFO: 49epoch:train:1-551batch: iter_time=0.002, forward_time=0.093, loss_ctc=0.831, loss=0.831, backward_time=0.117, optim_step_time=0.059, optim0_lr0=2.571e-04, train_time=0.986 +[ALab02] 2023-06-11 08:27:44,514 (trainer:721) INFO: 49epoch:train:552-1102batch: iter_time=3.955e-04, forward_time=0.092, loss_ctc=0.815, loss=0.815, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.569e-04, train_time=0.976 +[ALab02] 2023-06-11 08:30:02,868 (trainer:721) INFO: 49epoch:train:1103-1653batch: iter_time=3.470e-04, forward_time=0.094, loss_ctc=0.802, loss=0.802, backward_time=0.122, optim_step_time=0.059, optim0_lr0=2.568e-04, train_time=1.004 +[ALab02] 2023-06-11 08:32:32,670 (trainer:721) INFO: 49epoch:train:1654-2204batch: iter_time=3.986e-04, forward_time=0.103, loss_ctc=0.835, loss=0.835, backward_time=0.133, optim_step_time=0.062, optim0_lr0=2.567e-04, train_time=1.087 +[ALab02] 2023-06-11 08:34:49,718 (trainer:721) INFO: 49epoch:train:2205-2755batch: iter_time=3.446e-04, forward_time=0.094, loss_ctc=0.805, loss=0.805, backward_time=0.119, optim_step_time=0.059, optim0_lr0=2.565e-04, train_time=0.994 +[ALab02] 2023-06-11 08:37:05,971 (trainer:721) INFO: 49epoch:train:2756-3306batch: iter_time=3.481e-04, forward_time=0.093, loss_ctc=0.832, loss=0.832, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.564e-04, train_time=0.989 +[ALab02] 2023-06-11 08:39:21,469 (trainer:721) INFO: 49epoch:train:3307-3857batch: iter_time=3.670e-04, forward_time=0.093, loss_ctc=0.860, loss=0.860, backward_time=0.117, optim_step_time=0.059, optim0_lr0=2.563e-04, train_time=0.983 +[ALab02] 2023-06-11 08:41:34,814 (trainer:721) INFO: 49epoch:train:3858-4408batch: iter_time=3.328e-04, forward_time=0.092, loss_ctc=0.867, loss=0.867, backward_time=0.115, optim_step_time=0.060, optim0_lr0=2.561e-04, train_time=0.967 +[ALab02] 2023-06-11 08:43:52,573 (trainer:721) INFO: 49epoch:train:4409-4959batch: iter_time=3.441e-04, forward_time=0.093, loss_ctc=0.813, loss=0.813, backward_time=0.122, optim_step_time=0.058, optim0_lr0=2.560e-04, train_time=0.999 +[ALab02] 2023-06-11 08:46:09,952 (trainer:721) INFO: 49epoch:train:4960-5510batch: iter_time=3.532e-04, forward_time=0.094, loss_ctc=0.823, loss=0.823, backward_time=0.120, optim_step_time=0.060, optim0_lr0=2.559e-04, train_time=0.998 +[ALab02] 2023-06-11 08:48:29,700 (trainer:721) INFO: 49epoch:train:5511-6061batch: iter_time=3.400e-04, forward_time=0.096, loss_ctc=0.795, loss=0.795, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.557e-04, train_time=1.013 +[ALab02] 2023-06-11 08:50:41,426 (trainer:721) INFO: 49epoch:train:6062-6612batch: iter_time=3.235e-04, forward_time=0.087, loss_ctc=0.802, loss=0.802, backward_time=0.118, optim_step_time=0.054, optim0_lr0=2.556e-04, train_time=0.955 +[ALab02] 2023-06-11 08:52:56,795 (trainer:721) INFO: 49epoch:train:6613-7163batch: iter_time=3.363e-04, forward_time=0.095, loss_ctc=0.792, loss=0.792, backward_time=0.116, optim_step_time=0.060, optim0_lr0=2.555e-04, train_time=0.984 +[ALab02] 2023-06-11 08:55:06,365 (trainer:721) INFO: 49epoch:train:7164-7714batch: iter_time=3.348e-04, forward_time=0.088, loss_ctc=0.769, loss=0.769, backward_time=0.114, optim_step_time=0.057, optim0_lr0=2.553e-04, train_time=0.940 +[ALab02] 2023-06-11 08:57:13,529 (trainer:721) INFO: 49epoch:train:7715-8265batch: iter_time=3.176e-04, forward_time=0.087, loss_ctc=0.834, loss=0.834, backward_time=0.111, optim_step_time=0.054, optim0_lr0=2.552e-04, train_time=0.923 +[ALab02] 2023-06-11 08:59:16,191 (trainer:721) INFO: 49epoch:train:8266-8816batch: iter_time=3.143e-04, forward_time=0.084, loss_ctc=0.820, loss=0.820, backward_time=0.105, optim_step_time=0.053, optim0_lr0=2.551e-04, train_time=0.890 +[ALab02] 2023-06-11 09:01:17,663 (trainer:721) INFO: 49epoch:train:8817-9367batch: iter_time=3.231e-04, forward_time=0.083, loss_ctc=0.795, loss=0.795, backward_time=0.106, optim_step_time=0.053, optim0_lr0=2.550e-04, train_time=0.882 +[ALab02] 2023-06-11 09:03:23,195 (trainer:721) INFO: 49epoch:train:9368-9918batch: iter_time=3.254e-04, forward_time=0.085, loss_ctc=0.790, loss=0.790, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.548e-04, train_time=0.911 +[ALab02] 2023-06-11 09:05:24,816 (trainer:721) INFO: 49epoch:train:9919-10469batch: iter_time=3.327e-04, forward_time=0.082, loss_ctc=0.765, loss=0.765, backward_time=0.107, optim_step_time=0.052, optim0_lr0=2.547e-04, train_time=0.882 +[ALab02] 2023-06-11 09:07:25,833 (trainer:721) INFO: 49epoch:train:10470-11020batch: iter_time=3.114e-04, forward_time=0.081, loss_ctc=0.789, loss=0.789, backward_time=0.107, optim_step_time=0.052, optim0_lr0=2.546e-04, train_time=0.878 +[ALab02] 2023-06-11 09:08:38,957 (trainer:338) INFO: 49epoch results: [train] iter_time=4.301e-04, forward_time=0.091, loss_ctc=0.811, loss=0.811, backward_time=0.116, optim_step_time=0.057, optim0_lr0=2.558e-04, train_time=0.962, time=44 minutes and 14.11 seconds, total_count=540421, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.392, cer_ctc=0.054, cer=0.054, loss=4.392, time=31.14 seconds, total_count=21609, gpu_max_cached_mem_GB=10.861, [att_plot] time=39.66 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 09:08:42,078 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 09:08:42,082 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/35epoch.pth +[ALab02] 2023-06-11 09:08:42,082 (trainer:272) INFO: 50/70epoch started. Estimated time to finish: 19 hours, 4 minutes and 51.37 seconds +[ALab02] 2023-06-11 09:10:48,229 (trainer:721) INFO: 50epoch:train:1-551batch: iter_time=0.002, forward_time=0.085, loss_ctc=0.759, loss=0.759, backward_time=0.111, optim_step_time=0.051, optim0_lr0=2.544e-04, train_time=0.916 +[ALab02] 2023-06-11 09:12:53,245 (trainer:721) INFO: 50epoch:train:552-1102batch: iter_time=3.647e-04, forward_time=0.086, loss_ctc=0.794, loss=0.794, backward_time=0.109, optim_step_time=0.052, optim0_lr0=2.543e-04, train_time=0.906 +[ALab02] 2023-06-11 09:14:54,204 (trainer:721) INFO: 50epoch:train:1103-1653batch: iter_time=3.238e-04, forward_time=0.082, loss_ctc=0.767, loss=0.767, backward_time=0.106, optim_step_time=0.050, optim0_lr0=2.542e-04, train_time=0.878 +[ALab02] 2023-06-11 09:16:55,077 (trainer:721) INFO: 50epoch:train:1654-2204batch: iter_time=3.115e-04, forward_time=0.081, loss_ctc=0.758, loss=0.758, backward_time=0.107, optim_step_time=0.048, optim0_lr0=2.540e-04, train_time=0.877 +[ALab02] 2023-06-11 09:18:55,619 (trainer:721) INFO: 50epoch:train:2205-2755batch: iter_time=3.526e-04, forward_time=0.082, loss_ctc=0.773, loss=0.773, backward_time=0.106, optim_step_time=0.048, optim0_lr0=2.539e-04, train_time=0.876 +[ALab02] 2023-06-11 09:20:55,144 (trainer:721) INFO: 50epoch:train:2756-3306batch: iter_time=3.110e-04, forward_time=0.081, loss_ctc=0.803, loss=0.803, backward_time=0.106, optim_step_time=0.048, optim0_lr0=2.538e-04, train_time=0.867 +[ALab02] 2023-06-11 09:23:00,287 (trainer:721) INFO: 50epoch:train:3307-3857batch: iter_time=3.140e-04, forward_time=0.085, loss_ctc=0.747, loss=0.747, backward_time=0.110, optim_step_time=0.052, optim0_lr0=2.537e-04, train_time=0.908 +[ALab02] 2023-06-11 09:25:09,025 (trainer:721) INFO: 50epoch:train:3858-4408batch: iter_time=3.306e-04, forward_time=0.088, loss_ctc=0.803, loss=0.803, backward_time=0.112, optim_step_time=0.055, optim0_lr0=2.535e-04, train_time=0.934 +[ALab02] 2023-06-11 09:27:16,742 (trainer:721) INFO: 50epoch:train:4409-4959batch: iter_time=3.196e-04, forward_time=0.088, loss_ctc=0.772, loss=0.772, backward_time=0.111, optim_step_time=0.055, optim0_lr0=2.534e-04, train_time=0.927 +[ALab02] 2023-06-11 09:29:24,617 (trainer:721) INFO: 50epoch:train:4960-5510batch: iter_time=3.150e-04, forward_time=0.088, loss_ctc=0.759, loss=0.759, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.533e-04, train_time=0.928 +[ALab02] 2023-06-11 09:31:31,022 (trainer:721) INFO: 50epoch:train:5511-6061batch: iter_time=3.172e-04, forward_time=0.087, loss_ctc=0.758, loss=0.758, backward_time=0.111, optim_step_time=0.052, optim0_lr0=2.531e-04, train_time=0.916 +[ALab02] 2023-06-11 09:33:39,850 (trainer:721) INFO: 50epoch:train:6062-6612batch: iter_time=3.225e-04, forward_time=0.089, loss_ctc=0.744, loss=0.744, backward_time=0.112, optim_step_time=0.055, optim0_lr0=2.530e-04, train_time=0.935 +[ALab02] 2023-06-11 09:35:52,338 (trainer:721) INFO: 50epoch:train:6613-7163batch: iter_time=3.532e-04, forward_time=0.092, loss_ctc=0.774, loss=0.774, backward_time=0.114, optim_step_time=0.056, optim0_lr0=2.529e-04, train_time=0.962 +[ALab02] 2023-06-11 09:37:59,330 (trainer:721) INFO: 50epoch:train:7164-7714batch: iter_time=3.272e-04, forward_time=0.088, loss_ctc=0.767, loss=0.767, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.528e-04, train_time=0.921 +[ALab02] 2023-06-11 09:40:06,198 (trainer:721) INFO: 50epoch:train:7715-8265batch: iter_time=3.451e-04, forward_time=0.088, loss_ctc=0.758, loss=0.758, backward_time=0.109, optim_step_time=0.054, optim0_lr0=2.526e-04, train_time=0.920 +[ALab02] 2023-06-11 09:42:16,806 (trainer:721) INFO: 50epoch:train:8266-8816batch: iter_time=3.422e-04, forward_time=0.089, loss_ctc=0.770, loss=0.770, backward_time=0.114, optim_step_time=0.056, optim0_lr0=2.525e-04, train_time=0.948 +[ALab02] 2023-06-11 09:44:22,343 (trainer:721) INFO: 50epoch:train:8817-9367batch: iter_time=3.155e-04, forward_time=0.086, loss_ctc=0.731, loss=0.731, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.524e-04, train_time=0.911 +[ALab02] 2023-06-11 09:46:28,803 (trainer:721) INFO: 50epoch:train:9368-9918batch: iter_time=3.262e-04, forward_time=0.086, loss_ctc=0.724, loss=0.724, backward_time=0.111, optim_step_time=0.052, optim0_lr0=2.523e-04, train_time=0.917 +[ALab02] 2023-06-11 09:48:30,991 (trainer:721) INFO: 50epoch:train:9919-10469batch: iter_time=3.110e-04, forward_time=0.083, loss_ctc=0.787, loss=0.787, backward_time=0.106, optim_step_time=0.052, optim0_lr0=2.521e-04, train_time=0.887 +[ALab02] 2023-06-11 09:50:42,601 (trainer:721) INFO: 50epoch:train:10470-11020batch: iter_time=3.278e-04, forward_time=0.090, loss_ctc=0.771, loss=0.771, backward_time=0.116, optim_step_time=0.056, optim0_lr0=2.520e-04, train_time=0.955 +[ALab02] 2023-06-11 09:51:59,772 (trainer:338) INFO: 50epoch results: [train] iter_time=4.018e-04, forward_time=0.086, loss_ctc=0.766, loss=0.766, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.532e-04, train_time=0.915, time=42 minutes and 3.2 seconds, total_count=551450, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.337, cer_ctc=0.054, cer=0.054, loss=4.337, time=31.32 seconds, total_count=22050, gpu_max_cached_mem_GB=10.861, [att_plot] time=43.16 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 09:52:02,955 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 09:52:02,960 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/40epoch.pth +[ALab02] 2023-06-11 09:52:02,960 (trainer:272) INFO: 51/70epoch started. Estimated time to finish: 18 hours, 5 minutes and 52.29 seconds +[ALab02] 2023-06-11 09:54:08,311 (trainer:721) INFO: 51epoch:train:1-551batch: iter_time=0.002, forward_time=0.083, loss_ctc=0.734, loss=0.734, backward_time=0.110, optim_step_time=0.050, optim0_lr0=2.519e-04, train_time=0.910 +[ALab02] 2023-06-11 09:56:12,770 (trainer:721) INFO: 51epoch:train:552-1102batch: iter_time=3.336e-04, forward_time=0.085, loss_ctc=0.762, loss=0.762, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.518e-04, train_time=0.903 +[ALab02] 2023-06-11 09:58:21,233 (trainer:721) INFO: 51epoch:train:1103-1653batch: iter_time=3.286e-04, forward_time=0.088, loss_ctc=0.739, loss=0.739, backward_time=0.112, optim_step_time=0.054, optim0_lr0=2.516e-04, train_time=0.932 +[ALab02] 2023-06-11 10:00:36,345 (trainer:721) INFO: 51epoch:train:1654-2204batch: iter_time=3.683e-04, forward_time=0.093, loss_ctc=0.738, loss=0.738, backward_time=0.119, optim_step_time=0.056, optim0_lr0=2.515e-04, train_time=0.980 +[ALab02] 2023-06-11 10:02:53,694 (trainer:721) INFO: 51epoch:train:2205-2755batch: iter_time=3.211e-04, forward_time=0.095, loss_ctc=0.771, loss=0.771, backward_time=0.120, optim_step_time=0.059, optim0_lr0=2.514e-04, train_time=0.997 +[ALab02] 2023-06-11 10:05:10,433 (trainer:721) INFO: 51epoch:train:2756-3306batch: iter_time=3.495e-04, forward_time=0.094, loss_ctc=0.732, loss=0.732, backward_time=0.121, optim_step_time=0.056, optim0_lr0=2.513e-04, train_time=0.992 +[ALab02] 2023-06-11 10:07:25,808 (trainer:721) INFO: 51epoch:train:3307-3857batch: iter_time=3.291e-04, forward_time=0.091, loss_ctc=0.741, loss=0.741, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.511e-04, train_time=0.983 +[ALab02] 2023-06-11 10:09:49,332 (trainer:721) INFO: 51epoch:train:3858-4408batch: iter_time=3.562e-04, forward_time=0.098, loss_ctc=0.764, loss=0.764, backward_time=0.127, optim_step_time=0.061, optim0_lr0=2.510e-04, train_time=1.041 +[ALab02] 2023-06-11 10:12:17,592 (trainer:721) INFO: 51epoch:train:4409-4959batch: iter_time=4.090e-04, forward_time=0.102, loss_ctc=0.766, loss=0.766, backward_time=0.130, optim_step_time=0.064, optim0_lr0=2.509e-04, train_time=1.076 +[ALab02] 2023-06-11 10:14:32,776 (trainer:721) INFO: 51epoch:train:4960-5510batch: iter_time=3.232e-04, forward_time=0.096, loss_ctc=0.769, loss=0.769, backward_time=0.114, optim_step_time=0.059, optim0_lr0=2.508e-04, train_time=0.982 +[ALab02] 2023-06-11 10:16:46,937 (trainer:721) INFO: 51epoch:train:5511-6061batch: iter_time=3.251e-04, forward_time=0.093, loss_ctc=0.762, loss=0.762, backward_time=0.116, optim_step_time=0.058, optim0_lr0=2.506e-04, train_time=0.974 +[ALab02] 2023-06-11 10:18:58,479 (trainer:721) INFO: 51epoch:train:6062-6612batch: iter_time=3.171e-04, forward_time=0.091, loss_ctc=0.784, loss=0.784, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.505e-04, train_time=0.954 +[ALab02] 2023-06-11 10:21:07,002 (trainer:721) INFO: 51epoch:train:6613-7163batch: iter_time=3.144e-04, forward_time=0.089, loss_ctc=0.770, loss=0.770, backward_time=0.112, optim_step_time=0.054, optim0_lr0=2.504e-04, train_time=0.933 +[ALab02] 2023-06-11 10:23:14,726 (trainer:721) INFO: 51epoch:train:7164-7714batch: iter_time=3.126e-04, forward_time=0.087, loss_ctc=0.783, loss=0.783, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.503e-04, train_time=0.927 +[ALab02] 2023-06-11 10:25:17,274 (trainer:721) INFO: 51epoch:train:7715-8265batch: iter_time=3.041e-04, forward_time=0.083, loss_ctc=0.768, loss=0.768, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.501e-04, train_time=0.889 +[ALab02] 2023-06-11 10:27:19,618 (trainer:721) INFO: 51epoch:train:8266-8816batch: iter_time=3.089e-04, forward_time=0.083, loss_ctc=0.770, loss=0.770, backward_time=0.107, optim_step_time=0.051, optim0_lr0=2.500e-04, train_time=0.887 +[ALab02] 2023-06-11 10:29:17,135 (trainer:721) INFO: 51epoch:train:8817-9367batch: iter_time=2.901e-04, forward_time=0.079, loss_ctc=0.783, loss=0.783, backward_time=0.104, optim_step_time=0.047, optim0_lr0=2.499e-04, train_time=0.853 +[ALab02] 2023-06-11 10:31:21,672 (trainer:721) INFO: 51epoch:train:9368-9918batch: iter_time=3.144e-04, forward_time=0.084, loss_ctc=0.802, loss=0.802, backward_time=0.110, optim_step_time=0.052, optim0_lr0=2.498e-04, train_time=0.904 +[ALab02] 2023-06-11 10:33:23,497 (trainer:721) INFO: 51epoch:train:9919-10469batch: iter_time=3.005e-04, forward_time=0.083, loss_ctc=0.790, loss=0.790, backward_time=0.106, optim_step_time=0.051, optim0_lr0=2.496e-04, train_time=0.884 +[ALab02] 2023-06-11 10:35:26,375 (trainer:721) INFO: 51epoch:train:10470-11020batch: iter_time=3.057e-04, forward_time=0.084, loss_ctc=0.758, loss=0.758, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.495e-04, train_time=0.892 +[ALab02] 2023-06-11 10:36:50,563 (trainer:338) INFO: 51epoch results: [train] iter_time=4.271e-04, forward_time=0.089, loss_ctc=0.764, loss=0.764, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.507e-04, train_time=0.945, time=43 minutes and 25.96 seconds, total_count=562479, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.438, cer_ctc=0.054, cer=0.054, loss=4.438, time=34.36 seconds, total_count=22491, gpu_max_cached_mem_GB=10.861, [att_plot] time=47.28 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 10:36:53,714 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 10:36:53,717 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/48epoch.pth +[ALab02] 2023-06-11 10:36:53,718 (trainer:272) INFO: 52/70epoch started. Estimated time to finish: 17 hours, 8 minutes and 3.5 seconds +[ALab02] 2023-06-11 10:38:56,291 (trainer:721) INFO: 52epoch:train:1-551batch: iter_time=0.002, forward_time=0.082, loss_ctc=0.733, loss=0.733, backward_time=0.106, optim_step_time=0.051, optim0_lr0=2.494e-04, train_time=0.890 +[ALab02] 2023-06-11 10:40:59,734 (trainer:721) INFO: 52epoch:train:552-1102batch: iter_time=3.307e-04, forward_time=0.083, loss_ctc=0.748, loss=0.748, backward_time=0.108, optim_step_time=0.053, optim0_lr0=2.493e-04, train_time=0.895 +[ALab02] 2023-06-11 10:43:04,010 (trainer:721) INFO: 52epoch:train:1103-1653batch: iter_time=3.363e-04, forward_time=0.084, loss_ctc=0.739, loss=0.739, backward_time=0.110, optim_step_time=0.052, optim0_lr0=2.492e-04, train_time=0.902 +[ALab02] 2023-06-11 10:45:10,911 (trainer:721) INFO: 52epoch:train:1654-2204batch: iter_time=3.256e-04, forward_time=0.087, loss_ctc=0.753, loss=0.753, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.490e-04, train_time=0.921 +[ALab02] 2023-06-11 10:47:14,451 (trainer:721) INFO: 52epoch:train:2205-2755batch: iter_time=3.189e-04, forward_time=0.085, loss_ctc=0.753, loss=0.753, backward_time=0.107, optim_step_time=0.053, optim0_lr0=2.489e-04, train_time=0.898 +[ALab02] 2023-06-11 10:49:16,920 (trainer:721) INFO: 52epoch:train:2756-3306batch: iter_time=3.058e-04, forward_time=0.082, loss_ctc=0.733, loss=0.733, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.488e-04, train_time=0.888 +[ALab02] 2023-06-11 10:51:21,044 (trainer:721) INFO: 52epoch:train:3307-3857batch: iter_time=3.134e-04, forward_time=0.085, loss_ctc=0.718, loss=0.718, backward_time=0.107, optim_step_time=0.053, optim0_lr0=2.487e-04, train_time=0.900 +[ALab02] 2023-06-11 10:53:26,294 (trainer:721) INFO: 52epoch:train:3858-4408batch: iter_time=3.004e-04, forward_time=0.085, loss_ctc=0.760, loss=0.760, backward_time=0.109, optim_step_time=0.054, optim0_lr0=2.485e-04, train_time=0.909 +[ALab02] 2023-06-11 10:55:29,670 (trainer:721) INFO: 52epoch:train:4409-4959batch: iter_time=3.345e-04, forward_time=0.084, loss_ctc=0.714, loss=0.714, backward_time=0.107, optim_step_time=0.052, optim0_lr0=2.484e-04, train_time=0.895 +[ALab02] 2023-06-11 10:57:32,546 (trainer:721) INFO: 52epoch:train:4960-5510batch: iter_time=3.166e-04, forward_time=0.084, loss_ctc=0.758, loss=0.758, backward_time=0.107, optim_step_time=0.052, optim0_lr0=2.483e-04, train_time=0.892 +[ALab02] 2023-06-11 10:59:35,717 (trainer:721) INFO: 52epoch:train:5511-6061batch: iter_time=3.034e-04, forward_time=0.085, loss_ctc=0.732, loss=0.732, backward_time=0.107, optim_step_time=0.051, optim0_lr0=2.482e-04, train_time=0.894 +[ALab02] 2023-06-11 11:01:36,303 (trainer:721) INFO: 52epoch:train:6062-6612batch: iter_time=3.291e-04, forward_time=0.080, loss_ctc=0.749, loss=0.749, backward_time=0.107, optim_step_time=0.050, optim0_lr0=2.481e-04, train_time=0.875 +[ALab02] 2023-06-11 11:03:36,403 (trainer:721) INFO: 52epoch:train:6613-7163batch: iter_time=3.075e-04, forward_time=0.081, loss_ctc=0.766, loss=0.766, backward_time=0.105, optim_step_time=0.051, optim0_lr0=2.479e-04, train_time=0.872 +[ALab02] 2023-06-11 11:05:35,480 (trainer:721) INFO: 52epoch:train:7164-7714batch: iter_time=3.189e-04, forward_time=0.082, loss_ctc=0.718, loss=0.718, backward_time=0.103, optim_step_time=0.049, optim0_lr0=2.478e-04, train_time=0.865 +[ALab02] 2023-06-11 11:07:38,720 (trainer:721) INFO: 52epoch:train:7715-8265batch: iter_time=3.091e-04, forward_time=0.084, loss_ctc=0.769, loss=0.769, backward_time=0.107, optim_step_time=0.052, optim0_lr0=2.477e-04, train_time=0.893 +[ALab02] 2023-06-11 11:09:38,450 (trainer:721) INFO: 52epoch:train:8266-8816batch: iter_time=3.125e-04, forward_time=0.081, loss_ctc=0.737, loss=0.737, backward_time=0.105, optim_step_time=0.051, optim0_lr0=2.476e-04, train_time=0.869 +[ALab02] 2023-06-11 11:11:39,127 (trainer:721) INFO: 52epoch:train:8817-9367batch: iter_time=3.030e-04, forward_time=0.083, loss_ctc=0.741, loss=0.741, backward_time=0.105, optim_step_time=0.049, optim0_lr0=2.475e-04, train_time=0.877 +[ALab02] 2023-06-11 11:13:39,339 (trainer:721) INFO: 52epoch:train:9368-9918batch: iter_time=2.941e-04, forward_time=0.081, loss_ctc=0.676, loss=0.676, backward_time=0.105, optim_step_time=0.049, optim0_lr0=2.473e-04, train_time=0.872 +[ALab02] 2023-06-11 11:15:41,449 (trainer:721) INFO: 52epoch:train:9919-10469batch: iter_time=3.111e-04, forward_time=0.083, loss_ctc=0.720, loss=0.720, backward_time=0.106, optim_step_time=0.051, optim0_lr0=2.472e-04, train_time=0.886 +[ALab02] 2023-06-11 11:17:40,475 (trainer:721) INFO: 52epoch:train:10470-11020batch: iter_time=3.338e-04, forward_time=0.081, loss_ctc=0.742, loss=0.742, backward_time=0.103, optim_step_time=0.049, optim0_lr0=2.471e-04, train_time=0.864 +[ALab02] 2023-06-11 11:18:51,939 (trainer:338) INFO: 52epoch results: [train] iter_time=3.854e-04, forward_time=0.083, loss_ctc=0.738, loss=0.738, backward_time=0.107, optim_step_time=0.051, optim0_lr0=2.482e-04, train_time=0.888, time=40 minutes and 49.17 seconds, total_count=573508, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.309, cer_ctc=0.053, cer=0.053, loss=4.309, time=29.75 seconds, total_count=22932, gpu_max_cached_mem_GB=10.861, [att_plot] time=39.29 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 11:18:54,866 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 11:18:54,867 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/51epoch.pth +[ALab02] 2023-06-11 11:18:54,867 (trainer:272) INFO: 53/70epoch started. Estimated time to finish: 16 hours, 9 minutes and 45.91 seconds +[ALab02] 2023-06-11 11:20:48,147 (trainer:721) INFO: 53epoch:train:1-551batch: iter_time=0.002, forward_time=0.075, loss_ctc=0.706, loss=0.706, backward_time=0.099, optim_step_time=0.046, optim0_lr0=2.470e-04, train_time=0.822 +[ALab02] 2023-06-11 11:22:56,367 (trainer:721) INFO: 53epoch:train:552-1102batch: iter_time=3.713e-04, forward_time=0.091, loss_ctc=0.700, loss=0.700, backward_time=0.107, optim_step_time=0.049, optim0_lr0=2.469e-04, train_time=0.929 +[ALab02] 2023-06-11 11:25:09,743 (trainer:721) INFO: 53epoch:train:1103-1653batch: iter_time=3.230e-04, forward_time=0.097, loss_ctc=0.719, loss=0.719, backward_time=0.109, optim_step_time=0.051, optim0_lr0=2.468e-04, train_time=0.969 +[ALab02] 2023-06-11 11:27:14,879 (trainer:721) INFO: 53epoch:train:1654-2204batch: iter_time=2.993e-04, forward_time=0.091, loss_ctc=0.711, loss=0.711, backward_time=0.102, optim_step_time=0.048, optim0_lr0=2.466e-04, train_time=0.908 +[ALab02] 2023-06-11 11:29:10,624 (trainer:721) INFO: 53epoch:train:2205-2755batch: iter_time=2.784e-04, forward_time=0.081, loss_ctc=0.732, loss=0.732, backward_time=0.098, optim_step_time=0.046, optim0_lr0=2.465e-04, train_time=0.841 +[ALab02] 2023-06-11 11:31:11,782 (trainer:721) INFO: 53epoch:train:2756-3306batch: iter_time=2.773e-04, forward_time=0.088, loss_ctc=0.703, loss=0.703, backward_time=0.099, optim_step_time=0.044, optim0_lr0=2.464e-04, train_time=0.879 +[ALab02] 2023-06-11 11:33:23,341 (trainer:721) INFO: 53epoch:train:3307-3857batch: iter_time=3.046e-04, forward_time=0.095, loss_ctc=0.738, loss=0.738, backward_time=0.108, optim_step_time=0.050, optim0_lr0=2.463e-04, train_time=0.954 +[ALab02] 2023-06-11 11:35:27,712 (trainer:721) INFO: 53epoch:train:3858-4408batch: iter_time=2.912e-04, forward_time=0.091, loss_ctc=0.718, loss=0.718, backward_time=0.101, optim_step_time=0.046, optim0_lr0=2.462e-04, train_time=0.902 +[ALab02] 2023-06-11 11:37:26,647 (trainer:721) INFO: 53epoch:train:4409-4959batch: iter_time=2.646e-04, forward_time=0.085, loss_ctc=0.727, loss=0.727, backward_time=0.098, optim_step_time=0.043, optim0_lr0=2.460e-04, train_time=0.863 +[ALab02] 2023-06-11 11:39:17,823 (trainer:721) INFO: 53epoch:train:4960-5510batch: iter_time=2.400e-04, forward_time=0.080, loss_ctc=0.759, loss=0.759, backward_time=0.090, optim_step_time=0.040, optim0_lr0=2.459e-04, train_time=0.808 +[ALab02] 2023-06-11 11:41:19,373 (trainer:721) INFO: 53epoch:train:5511-6061batch: iter_time=2.476e-04, forward_time=0.088, loss_ctc=0.742, loss=0.742, backward_time=0.099, optim_step_time=0.045, optim0_lr0=2.458e-04, train_time=0.880 +[ALab02] 2023-06-11 11:43:12,821 (trainer:721) INFO: 53epoch:train:6062-6612batch: iter_time=2.598e-04, forward_time=0.082, loss_ctc=0.726, loss=0.726, backward_time=0.091, optim_step_time=0.041, optim0_lr0=2.457e-04, train_time=0.825 +[ALab02] 2023-06-11 11:45:05,338 (trainer:721) INFO: 53epoch:train:6613-7163batch: iter_time=2.556e-04, forward_time=0.081, loss_ctc=0.731, loss=0.731, backward_time=0.091, optim_step_time=0.040, optim0_lr0=2.456e-04, train_time=0.816 +[ALab02] 2023-06-11 11:46:56,911 (trainer:721) INFO: 53epoch:train:7164-7714batch: iter_time=2.232e-04, forward_time=0.080, loss_ctc=0.739, loss=0.739, backward_time=0.090, optim_step_time=0.041, optim0_lr0=2.455e-04, train_time=0.811 +[ALab02] 2023-06-11 11:49:02,431 (trainer:721) INFO: 53epoch:train:7715-8265batch: iter_time=2.715e-04, forward_time=0.091, loss_ctc=0.728, loss=0.728, backward_time=0.102, optim_step_time=0.047, optim0_lr0=2.453e-04, train_time=0.910 +[ALab02] 2023-06-11 11:51:00,309 (trainer:721) INFO: 53epoch:train:8266-8816batch: iter_time=2.537e-04, forward_time=0.084, loss_ctc=0.752, loss=0.752, backward_time=0.096, optim_step_time=0.043, optim0_lr0=2.452e-04, train_time=0.856 +[ALab02] 2023-06-11 11:52:49,809 (trainer:721) INFO: 53epoch:train:8817-9367batch: iter_time=2.222e-04, forward_time=0.079, loss_ctc=0.749, loss=0.749, backward_time=0.088, optim_step_time=0.039, optim0_lr0=2.451e-04, train_time=0.796 +[ALab02] 2023-06-11 11:54:43,901 (trainer:721) INFO: 53epoch:train:9368-9918batch: iter_time=2.404e-04, forward_time=0.082, loss_ctc=0.745, loss=0.745, backward_time=0.093, optim_step_time=0.042, optim0_lr0=2.450e-04, train_time=0.827 +[ALab02] 2023-06-11 11:56:31,331 (trainer:721) INFO: 53epoch:train:9919-10469batch: iter_time=2.365e-04, forward_time=0.077, loss_ctc=0.678, loss=0.678, backward_time=0.086, optim_step_time=0.039, optim0_lr0=2.449e-04, train_time=0.779 +[ALab02] 2023-06-11 11:58:18,958 (trainer:721) INFO: 53epoch:train:10470-11020batch: iter_time=2.361e-04, forward_time=0.077, loss_ctc=0.690, loss=0.690, backward_time=0.086, optim_step_time=0.038, optim0_lr0=2.448e-04, train_time=0.781 +[ALab02] 2023-06-11 11:59:23,371 (trainer:338) INFO: 53epoch results: [train] iter_time=3.571e-04, forward_time=0.085, loss_ctc=0.724, loss=0.724, backward_time=0.097, optim_step_time=0.044, optim0_lr0=2.459e-04, train_time=0.858, time=39 minutes and 25.95 seconds, total_count=584537, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.273, cer_ctc=0.053, cer=0.053, loss=4.273, time=29.15 seconds, total_count=23373, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.41 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 11:59:27,354 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 11:59:27,357 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/47epoch.pth +[ALab02] 2023-06-11 11:59:27,358 (trainer:272) INFO: 54/70epoch started. Estimated time to finish: 15 hours, 11 minutes and 36.74 seconds +[ALab02] 2023-06-11 12:01:20,779 (trainer:721) INFO: 54epoch:train:1-551batch: iter_time=0.001, forward_time=0.081, loss_ctc=0.642, loss=0.642, backward_time=0.092, optim_step_time=0.041, optim0_lr0=2.446e-04, train_time=0.823 +[ALab02] 2023-06-11 12:03:19,052 (trainer:721) INFO: 54epoch:train:552-1102batch: iter_time=2.588e-04, forward_time=0.084, loss_ctc=0.704, loss=0.704, backward_time=0.097, optim_step_time=0.043, optim0_lr0=2.445e-04, train_time=0.859 +[ALab02] 2023-06-11 12:05:14,024 (trainer:721) INFO: 54epoch:train:1103-1653batch: iter_time=2.590e-04, forward_time=0.084, loss_ctc=0.703, loss=0.703, backward_time=0.092, optim_step_time=0.043, optim0_lr0=2.444e-04, train_time=0.834 +[ALab02] 2023-06-11 12:07:04,139 (trainer:721) INFO: 54epoch:train:1654-2204batch: iter_time=2.312e-04, forward_time=0.079, loss_ctc=0.693, loss=0.693, backward_time=0.089, optim_step_time=0.039, optim0_lr0=2.443e-04, train_time=0.799 +[ALab02] 2023-06-11 12:08:59,589 (trainer:721) INFO: 54epoch:train:2205-2755batch: iter_time=2.491e-04, forward_time=0.083, loss_ctc=0.694, loss=0.694, backward_time=0.095, optim_step_time=0.042, optim0_lr0=2.442e-04, train_time=0.837 +[ALab02] 2023-06-11 12:10:45,972 (trainer:721) INFO: 54epoch:train:2756-3306batch: iter_time=2.260e-04, forward_time=0.075, loss_ctc=0.717, loss=0.717, backward_time=0.087, optim_step_time=0.039, optim0_lr0=2.441e-04, train_time=0.774 +[ALab02] 2023-06-11 12:12:35,711 (trainer:721) INFO: 54epoch:train:3307-3857batch: iter_time=2.239e-04, forward_time=0.079, loss_ctc=0.709, loss=0.709, backward_time=0.088, optim_step_time=0.039, optim0_lr0=2.440e-04, train_time=0.796 +[ALab02] 2023-06-11 12:14:29,931 (trainer:721) INFO: 54epoch:train:3858-4408batch: iter_time=2.374e-04, forward_time=0.081, loss_ctc=0.708, loss=0.708, backward_time=0.095, optim_step_time=0.043, optim0_lr0=2.438e-04, train_time=0.829 +[ALab02] 2023-06-11 12:16:22,898 (trainer:721) INFO: 54epoch:train:4409-4959batch: iter_time=2.392e-04, forward_time=0.081, loss_ctc=0.702, loss=0.702, backward_time=0.091, optim_step_time=0.041, optim0_lr0=2.437e-04, train_time=0.821 +[ALab02] 2023-06-11 12:18:14,823 (trainer:721) INFO: 54epoch:train:4960-5510batch: iter_time=2.415e-04, forward_time=0.080, loss_ctc=0.679, loss=0.679, backward_time=0.091, optim_step_time=0.040, optim0_lr0=2.436e-04, train_time=0.812 +[ALab02] 2023-06-11 12:20:04,971 (trainer:721) INFO: 54epoch:train:5511-6061batch: iter_time=2.536e-04, forward_time=0.080, loss_ctc=0.698, loss=0.698, backward_time=0.089, optim_step_time=0.040, optim0_lr0=2.435e-04, train_time=0.798 +[ALab02] 2023-06-11 12:22:01,241 (trainer:721) INFO: 54epoch:train:6062-6612batch: iter_time=2.458e-04, forward_time=0.083, loss_ctc=0.727, loss=0.727, backward_time=0.095, optim_step_time=0.041, optim0_lr0=2.434e-04, train_time=0.844 +[ALab02] 2023-06-11 12:23:51,420 (trainer:721) INFO: 54epoch:train:6613-7163batch: iter_time=2.372e-04, forward_time=0.078, loss_ctc=0.743, loss=0.743, backward_time=0.090, optim_step_time=0.040, optim0_lr0=2.433e-04, train_time=0.801 +[ALab02] 2023-06-11 12:25:38,036 (trainer:721) INFO: 54epoch:train:7164-7714batch: iter_time=2.162e-04, forward_time=0.076, loss_ctc=0.690, loss=0.690, backward_time=0.087, optim_step_time=0.039, optim0_lr0=2.432e-04, train_time=0.774 +[ALab02] 2023-06-11 12:27:37,065 (trainer:721) INFO: 54epoch:train:7715-8265batch: iter_time=2.460e-04, forward_time=0.086, loss_ctc=0.713, loss=0.713, backward_time=0.097, optim_step_time=0.043, optim0_lr0=2.431e-04, train_time=0.862 +[ALab02] 2023-06-11 12:29:29,426 (trainer:721) INFO: 54epoch:train:8266-8816batch: iter_time=2.371e-04, forward_time=0.080, loss_ctc=0.704, loss=0.704, backward_time=0.092, optim_step_time=0.039, optim0_lr0=2.429e-04, train_time=0.816 +[ALab02] 2023-06-11 12:31:17,119 (trainer:721) INFO: 54epoch:train:8817-9367batch: iter_time=2.249e-04, forward_time=0.076, loss_ctc=0.741, loss=0.741, backward_time=0.088, optim_step_time=0.039, optim0_lr0=2.428e-04, train_time=0.782 +[ALab02] 2023-06-11 12:33:05,992 (trainer:721) INFO: 54epoch:train:9368-9918batch: iter_time=2.163e-04, forward_time=0.078, loss_ctc=0.719, loss=0.719, backward_time=0.088, optim_step_time=0.040, optim0_lr0=2.427e-04, train_time=0.790 +[ALab02] 2023-06-11 12:34:56,870 (trainer:721) INFO: 54epoch:train:9919-10469batch: iter_time=2.389e-04, forward_time=0.079, loss_ctc=0.757, loss=0.757, backward_time=0.091, optim_step_time=0.040, optim0_lr0=2.426e-04, train_time=0.804 +[ALab02] 2023-06-11 12:36:43,503 (trainer:721) INFO: 54epoch:train:10470-11020batch: iter_time=2.275e-04, forward_time=0.076, loss_ctc=0.720, loss=0.720, backward_time=0.086, optim_step_time=0.039, optim0_lr0=2.425e-04, train_time=0.774 +[ALab02] 2023-06-11 12:37:45,921 (trainer:338) INFO: 54epoch results: [train] iter_time=2.882e-04, forward_time=0.080, loss_ctc=0.708, loss=0.708, backward_time=0.091, optim_step_time=0.041, optim0_lr0=2.436e-04, train_time=0.811, time=37 minutes and 17.92 seconds, total_count=595566, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.395, cer_ctc=0.054, cer=0.054, loss=4.395, time=28.24 seconds, total_count=23814, gpu_max_cached_mem_GB=10.861, [att_plot] time=32.4 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 12:37:48,508 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 12:37:48,511 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/49epoch.pth +[ALab02] 2023-06-11 12:37:48,511 (trainer:272) INFO: 55/70epoch started. Estimated time to finish: 14 hours, 13 minutes and 27.79 seconds +[ALab02] 2023-06-11 12:39:31,479 (trainer:721) INFO: 55epoch:train:1-551batch: iter_time=0.001, forward_time=0.071, loss_ctc=0.724, loss=0.724, backward_time=0.084, optim_step_time=0.038, optim0_lr0=2.424e-04, train_time=0.748 +[ALab02] 2023-06-11 12:41:26,296 (trainer:721) INFO: 55epoch:train:552-1102batch: iter_time=3.252e-04, forward_time=0.082, loss_ctc=0.687, loss=0.687, backward_time=0.094, optim_step_time=0.041, optim0_lr0=2.423e-04, train_time=0.833 +[ALab02] 2023-06-11 12:43:14,094 (trainer:721) INFO: 55epoch:train:1103-1653batch: iter_time=2.547e-04, forward_time=0.075, loss_ctc=0.705, loss=0.705, backward_time=0.089, optim_step_time=0.040, optim0_lr0=2.422e-04, train_time=0.782 +[ALab02] 2023-06-11 12:44:58,564 (trainer:721) INFO: 55epoch:train:1654-2204batch: iter_time=2.074e-04, forward_time=0.074, loss_ctc=0.717, loss=0.717, backward_time=0.085, optim_step_time=0.037, optim0_lr0=2.420e-04, train_time=0.758 +[ALab02] 2023-06-11 12:46:44,604 (trainer:721) INFO: 55epoch:train:2205-2755batch: iter_time=1.975e-04, forward_time=0.075, loss_ctc=0.725, loss=0.725, backward_time=0.086, optim_step_time=0.040, optim0_lr0=2.419e-04, train_time=0.768 +[ALab02] 2023-06-11 12:48:29,802 (trainer:721) INFO: 55epoch:train:2756-3306batch: iter_time=2.174e-04, forward_time=0.074, loss_ctc=0.717, loss=0.717, backward_time=0.086, optim_step_time=0.038, optim0_lr0=2.418e-04, train_time=0.766 +[ALab02] 2023-06-11 12:50:15,466 (trainer:721) INFO: 55epoch:train:3307-3857batch: iter_time=2.396e-04, forward_time=0.075, loss_ctc=0.696, loss=0.696, backward_time=0.085, optim_step_time=0.038, optim0_lr0=2.417e-04, train_time=0.766 +[ALab02] 2023-06-11 12:51:58,363 (trainer:721) INFO: 55epoch:train:3858-4408batch: iter_time=1.849e-04, forward_time=0.073, loss_ctc=0.707, loss=0.707, backward_time=0.083, optim_step_time=0.037, optim0_lr0=2.416e-04, train_time=0.747 +[ALab02] 2023-06-11 12:53:45,695 (trainer:721) INFO: 55epoch:train:4409-4959batch: iter_time=2.244e-04, forward_time=0.076, loss_ctc=0.723, loss=0.723, backward_time=0.088, optim_step_time=0.038, optim0_lr0=2.415e-04, train_time=0.780 +[ALab02] 2023-06-11 12:55:31,687 (trainer:721) INFO: 55epoch:train:4960-5510batch: iter_time=2.181e-04, forward_time=0.076, loss_ctc=0.728, loss=0.728, backward_time=0.085, optim_step_time=0.038, optim0_lr0=2.414e-04, train_time=0.769 +[ALab02] 2023-06-11 12:57:12,492 (trainer:721) INFO: 55epoch:train:5511-6061batch: iter_time=1.980e-04, forward_time=0.071, loss_ctc=0.714, loss=0.714, backward_time=0.081, optim_step_time=0.037, optim0_lr0=2.413e-04, train_time=0.731 +[ALab02] 2023-06-11 12:59:03,055 (trainer:721) INFO: 55epoch:train:6062-6612batch: iter_time=2.131e-04, forward_time=0.080, loss_ctc=0.715, loss=0.715, backward_time=0.089, optim_step_time=0.041, optim0_lr0=2.412e-04, train_time=0.802 +[ALab02] 2023-06-11 13:01:53,785 (trainer:721) INFO: 55epoch:train:6613-7163batch: iter_time=3.566e-04, forward_time=0.125, loss_ctc=0.691, loss=0.691, backward_time=0.143, optim_step_time=0.066, optim0_lr0=2.410e-04, train_time=1.239 +[ALab02] 2023-06-11 13:04:36,278 (trainer:721) INFO: 55epoch:train:7164-7714batch: iter_time=3.653e-04, forward_time=0.119, loss_ctc=0.731, loss=0.731, backward_time=0.135, optim_step_time=0.064, optim0_lr0=2.409e-04, train_time=1.179 +[ALab02] 2023-06-11 13:07:19,141 (trainer:721) INFO: 55epoch:train:7715-8265batch: iter_time=3.563e-04, forward_time=0.120, loss_ctc=0.760, loss=0.760, backward_time=0.135, optim_step_time=0.064, optim0_lr0=2.408e-04, train_time=1.182 +[ALab02] 2023-06-11 13:09:55,382 (trainer:721) INFO: 55epoch:train:8266-8816batch: iter_time=3.328e-04, forward_time=0.113, loss_ctc=0.738, loss=0.738, backward_time=0.131, optim_step_time=0.061, optim0_lr0=2.407e-04, train_time=1.134 +[ALab02] 2023-06-11 13:12:29,402 (trainer:721) INFO: 55epoch:train:8817-9367batch: iter_time=3.438e-04, forward_time=0.113, loss_ctc=0.765, loss=0.765, backward_time=0.127, optim_step_time=0.061, optim0_lr0=2.406e-04, train_time=1.119 +[ALab02] 2023-06-11 13:15:08,567 (trainer:721) INFO: 55epoch:train:9368-9918batch: iter_time=3.637e-04, forward_time=0.118, loss_ctc=0.730, loss=0.730, backward_time=0.130, optim_step_time=0.063, optim0_lr0=2.405e-04, train_time=1.153 +[ALab02] 2023-06-11 13:17:42,839 (trainer:721) INFO: 55epoch:train:9919-10469batch: iter_time=3.515e-04, forward_time=0.114, loss_ctc=0.728, loss=0.728, backward_time=0.127, optim_step_time=0.059, optim0_lr0=2.404e-04, train_time=1.120 +[ALab02] 2023-06-11 13:20:13,662 (trainer:721) INFO: 55epoch:train:10470-11020batch: iter_time=3.485e-04, forward_time=0.112, loss_ctc=0.726, loss=0.726, backward_time=0.122, optim_step_time=0.059, optim0_lr0=2.403e-04, train_time=1.094 +[ALab02] 2023-06-11 13:21:48,355 (trainer:338) INFO: 55epoch results: [train] iter_time=3.267e-04, forward_time=0.092, loss_ctc=0.721, loss=0.721, backward_time=0.104, optim_step_time=0.048, optim0_lr0=2.413e-04, train_time=0.924, time=42 minutes and 27.97 seconds, total_count=606595, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.416, cer_ctc=0.054, cer=0.054, loss=4.416, time=34.48 seconds, total_count=24255, gpu_max_cached_mem_GB=10.861, [att_plot] time=57.39 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 13:21:52,087 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 13:21:52,090 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/44epoch.pth +[ALab02] 2023-06-11 13:21:52,090 (trainer:272) INFO: 56/70epoch started. Estimated time to finish: 13 hours, 17 minutes and 35.42 seconds +[ALab02] 2023-06-11 13:24:21,418 (trainer:721) INFO: 56epoch:train:1-551batch: iter_time=0.002, forward_time=0.107, loss_ctc=0.768, loss=0.768, backward_time=0.124, optim_step_time=0.057, optim0_lr0=2.402e-04, train_time=1.085 +[ALab02] 2023-06-11 13:26:42,552 (trainer:721) INFO: 56epoch:train:552-1102batch: iter_time=3.368e-04, forward_time=0.104, loss_ctc=0.708, loss=0.708, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.401e-04, train_time=1.023 +[ALab02] 2023-06-11 13:29:19,500 (trainer:721) INFO: 56epoch:train:1103-1653batch: iter_time=3.623e-04, forward_time=0.116, loss_ctc=0.714, loss=0.714, backward_time=0.129, optim_step_time=0.062, optim0_lr0=2.399e-04, train_time=1.138 +[ALab02] 2023-06-11 13:31:49,543 (trainer:721) INFO: 56epoch:train:1654-2204batch: iter_time=3.529e-04, forward_time=0.110, loss_ctc=0.713, loss=0.713, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.398e-04, train_time=1.088 +[ALab02] 2023-06-11 13:34:16,901 (trainer:721) INFO: 56epoch:train:2205-2755batch: iter_time=3.155e-04, forward_time=0.108, loss_ctc=0.733, loss=0.733, backward_time=0.121, optim_step_time=0.057, optim0_lr0=2.397e-04, train_time=1.070 +[ALab02] 2023-06-11 13:36:56,355 (trainer:721) INFO: 56epoch:train:2756-3306batch: iter_time=3.552e-04, forward_time=0.116, loss_ctc=0.746, loss=0.746, backward_time=0.133, optim_step_time=0.061, optim0_lr0=2.396e-04, train_time=1.157 +[ALab02] 2023-06-11 13:39:28,114 (trainer:721) INFO: 56epoch:train:3307-3857batch: iter_time=3.723e-04, forward_time=0.113, loss_ctc=0.716, loss=0.716, backward_time=0.124, optim_step_time=0.058, optim0_lr0=2.395e-04, train_time=1.101 +[ALab02] 2023-06-11 13:41:53,887 (trainer:721) INFO: 56epoch:train:3858-4408batch: iter_time=3.247e-04, forward_time=0.107, loss_ctc=0.734, loss=0.734, backward_time=0.120, optim_step_time=0.056, optim0_lr0=2.394e-04, train_time=1.057 +[ALab02] 2023-06-11 13:44:28,043 (trainer:721) INFO: 56epoch:train:4409-4959batch: iter_time=3.941e-04, forward_time=0.113, loss_ctc=0.687, loss=0.687, backward_time=0.128, optim_step_time=0.059, optim0_lr0=2.393e-04, train_time=1.119 +[ALab02] 2023-06-11 13:46:57,305 (trainer:721) INFO: 56epoch:train:4960-5510batch: iter_time=3.321e-04, forward_time=0.109, loss_ctc=0.687, loss=0.687, backward_time=0.124, optim_step_time=0.058, optim0_lr0=2.392e-04, train_time=1.083 +[ALab02] 2023-06-11 13:49:21,675 (trainer:721) INFO: 56epoch:train:5511-6061batch: iter_time=3.265e-04, forward_time=0.105, loss_ctc=0.726, loss=0.726, backward_time=0.119, optim_step_time=0.058, optim0_lr0=2.391e-04, train_time=1.047 +[ALab02] 2023-06-11 13:51:52,322 (trainer:721) INFO: 56epoch:train:6062-6612batch: iter_time=3.365e-04, forward_time=0.111, loss_ctc=0.724, loss=0.724, backward_time=0.124, optim_step_time=0.058, optim0_lr0=2.390e-04, train_time=1.093 +[ALab02] 2023-06-11 13:54:11,479 (trainer:721) INFO: 56epoch:train:6613-7163batch: iter_time=3.286e-04, forward_time=0.099, loss_ctc=0.752, loss=0.752, backward_time=0.118, optim_step_time=0.059, optim0_lr0=2.389e-04, train_time=1.011 +[ALab02] 2023-06-11 13:56:29,006 (trainer:721) INFO: 56epoch:train:7164-7714batch: iter_time=3.403e-04, forward_time=0.094, loss_ctc=0.724, loss=0.724, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.388e-04, train_time=0.997 +[ALab02] 2023-06-11 13:58:47,400 (trainer:721) INFO: 56epoch:train:7715-8265batch: iter_time=3.897e-04, forward_time=0.097, loss_ctc=0.721, loss=0.721, backward_time=0.119, optim_step_time=0.058, optim0_lr0=2.386e-04, train_time=1.004 +[ALab02] 2023-06-11 14:01:00,797 (trainer:721) INFO: 56epoch:train:8266-8816batch: iter_time=3.243e-04, forward_time=0.092, loss_ctc=0.726, loss=0.726, backward_time=0.116, optim_step_time=0.057, optim0_lr0=2.385e-04, train_time=0.968 +[ALab02] 2023-06-11 14:03:12,255 (trainer:721) INFO: 56epoch:train:8817-9367batch: iter_time=3.106e-04, forward_time=0.091, loss_ctc=0.716, loss=0.716, backward_time=0.114, optim_step_time=0.056, optim0_lr0=2.384e-04, train_time=0.955 +[ALab02] 2023-06-11 14:05:25,516 (trainer:721) INFO: 56epoch:train:9368-9918batch: iter_time=3.071e-04, forward_time=0.091, loss_ctc=0.723, loss=0.723, backward_time=0.118, optim_step_time=0.055, optim0_lr0=2.383e-04, train_time=0.967 +[ALab02] 2023-06-11 14:07:28,248 (trainer:721) INFO: 56epoch:train:9919-10469batch: iter_time=3.032e-04, forward_time=0.085, loss_ctc=0.736, loss=0.736, backward_time=0.105, optim_step_time=0.053, optim0_lr0=2.382e-04, train_time=0.890 +[ALab02] 2023-06-11 14:09:29,134 (trainer:721) INFO: 56epoch:train:10470-11020batch: iter_time=2.825e-04, forward_time=0.081, loss_ctc=0.765, loss=0.765, backward_time=0.107, optim_step_time=0.048, optim0_lr0=2.381e-04, train_time=0.877 +[ALab02] 2023-06-11 14:10:33,895 (trainer:338) INFO: 56epoch results: [train] iter_time=4.097e-04, forward_time=0.102, loss_ctc=0.726, loss=0.726, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.391e-04, train_time=1.036, time=47 minutes and 39.2 seconds, total_count=617624, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.476, cer_ctc=0.054, cer=0.054, loss=4.476, time=27.37 seconds, total_count=24696, gpu_max_cached_mem_GB=10.861, [att_plot] time=35.24 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 14:10:36,980 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 14:10:36,983 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/50epoch.pth +[ALab02] 2023-06-11 14:10:36,984 (trainer:272) INFO: 57/70epoch started. Estimated time to finish: 12 hours, 23 minutes and 18.69 seconds +[ALab02] 2023-06-11 14:12:41,010 (trainer:721) INFO: 57epoch:train:1-551batch: iter_time=0.001, forward_time=0.083, loss_ctc=0.707, loss=0.707, backward_time=0.109, optim_step_time=0.050, optim0_lr0=2.380e-04, train_time=0.901 +[ALab02] 2023-06-11 14:14:29,794 (trainer:721) INFO: 57epoch:train:552-1102batch: iter_time=2.959e-04, forward_time=0.072, loss_ctc=0.713, loss=0.713, backward_time=0.096, optim_step_time=0.044, optim0_lr0=2.379e-04, train_time=0.789 +[ALab02] 2023-06-11 14:16:09,743 (trainer:721) INFO: 57epoch:train:1103-1653batch: iter_time=2.437e-04, forward_time=0.066, loss_ctc=0.711, loss=0.711, backward_time=0.088, optim_step_time=0.040, optim0_lr0=2.378e-04, train_time=0.726 +[ALab02] 2023-06-11 14:17:45,214 (trainer:721) INFO: 57epoch:train:1654-2204batch: iter_time=2.583e-04, forward_time=0.062, loss_ctc=0.731, loss=0.731, backward_time=0.084, optim_step_time=0.040, optim0_lr0=2.377e-04, train_time=0.693 +[ALab02] 2023-06-11 14:19:23,657 (trainer:721) INFO: 57epoch:train:2205-2755batch: iter_time=2.507e-04, forward_time=0.064, loss_ctc=0.704, loss=0.704, backward_time=0.088, optim_step_time=0.041, optim0_lr0=2.376e-04, train_time=0.714 +[ALab02] 2023-06-11 14:20:58,402 (trainer:721) INFO: 57epoch:train:2756-3306batch: iter_time=2.331e-04, forward_time=0.061, loss_ctc=0.702, loss=0.702, backward_time=0.083, optim_step_time=0.039, optim0_lr0=2.375e-04, train_time=0.688 +[ALab02] 2023-06-11 14:22:42,829 (trainer:721) INFO: 57epoch:train:3307-3857batch: iter_time=2.494e-04, forward_time=0.068, loss_ctc=0.679, loss=0.679, backward_time=0.093, optim_step_time=0.044, optim0_lr0=2.374e-04, train_time=0.757 +[ALab02] 2023-06-11 14:25:16,619 (trainer:721) INFO: 57epoch:train:3858-4408batch: iter_time=3.516e-04, forward_time=0.105, loss_ctc=0.723, loss=0.723, backward_time=0.137, optim_step_time=0.065, optim0_lr0=2.373e-04, train_time=1.116 +[ALab02] 2023-06-11 14:27:49,348 (trainer:721) INFO: 57epoch:train:4409-4959batch: iter_time=3.784e-04, forward_time=0.105, loss_ctc=0.748, loss=0.748, backward_time=0.134, optim_step_time=0.064, optim0_lr0=2.372e-04, train_time=1.108 +[ALab02] 2023-06-11 14:30:19,513 (trainer:721) INFO: 57epoch:train:4960-5510batch: iter_time=3.714e-04, forward_time=0.105, loss_ctc=0.701, loss=0.701, backward_time=0.130, optim_step_time=0.065, optim0_lr0=2.371e-04, train_time=1.089 +[ALab02] 2023-06-11 14:32:38,962 (trainer:721) INFO: 57epoch:train:5511-6061batch: iter_time=3.510e-04, forward_time=0.095, loss_ctc=0.689, loss=0.689, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.370e-04, train_time=1.012 +[ALab02] 2023-06-11 14:35:05,474 (trainer:721) INFO: 57epoch:train:6062-6612batch: iter_time=3.994e-04, forward_time=0.104, loss_ctc=0.752, loss=0.752, backward_time=0.126, optim_step_time=0.062, optim0_lr0=2.368e-04, train_time=1.064 +[ALab02] 2023-06-11 14:37:28,531 (trainer:721) INFO: 57epoch:train:6613-7163batch: iter_time=3.498e-04, forward_time=0.098, loss_ctc=0.728, loss=0.728, backward_time=0.127, optim_step_time=0.060, optim0_lr0=2.367e-04, train_time=1.038 +[ALab02] 2023-06-11 14:39:55,654 (trainer:721) INFO: 57epoch:train:7164-7714batch: iter_time=3.660e-04, forward_time=0.099, loss_ctc=0.707, loss=0.707, backward_time=0.131, optim_step_time=0.061, optim0_lr0=2.366e-04, train_time=1.067 +[ALab02] 2023-06-11 14:42:30,440 (trainer:721) INFO: 57epoch:train:7715-8265batch: iter_time=3.714e-04, forward_time=0.107, loss_ctc=0.721, loss=0.721, backward_time=0.137, optim_step_time=0.064, optim0_lr0=2.365e-04, train_time=1.124 +[ALab02] 2023-06-11 14:44:59,411 (trainer:721) INFO: 57epoch:train:8266-8816batch: iter_time=3.670e-04, forward_time=0.102, loss_ctc=0.707, loss=0.707, backward_time=0.131, optim_step_time=0.064, optim0_lr0=2.364e-04, train_time=1.080 +[ALab02] 2023-06-11 14:47:28,437 (trainer:721) INFO: 57epoch:train:8817-9367batch: iter_time=3.774e-04, forward_time=0.102, loss_ctc=0.702, loss=0.702, backward_time=0.132, optim_step_time=0.062, optim0_lr0=2.363e-04, train_time=1.082 +[ALab02] 2023-06-11 14:49:55,155 (trainer:721) INFO: 57epoch:train:9368-9918batch: iter_time=3.666e-04, forward_time=0.101, loss_ctc=0.712, loss=0.712, backward_time=0.129, optim_step_time=0.061, optim0_lr0=2.362e-04, train_time=1.063 +[ALab02] 2023-06-11 14:52:18,245 (trainer:721) INFO: 57epoch:train:9919-10469batch: iter_time=3.499e-04, forward_time=0.098, loss_ctc=0.754, loss=0.754, backward_time=0.125, optim_step_time=0.061, optim0_lr0=2.361e-04, train_time=1.039 +[ALab02] 2023-06-11 14:54:37,279 (trainer:721) INFO: 57epoch:train:10470-11020batch: iter_time=3.367e-04, forward_time=0.095, loss_ctc=0.677, loss=0.677, backward_time=0.124, optim_step_time=0.057, optim0_lr0=2.360e-04, train_time=1.009 +[ALab02] 2023-06-11 14:56:14,391 (trainer:338) INFO: 57epoch results: [train] iter_time=3.797e-04, forward_time=0.090, loss_ctc=0.714, loss=0.714, backward_time=0.116, optim_step_time=0.055, optim0_lr0=2.370e-04, train_time=0.958, time=44 minutes and 2.89 seconds, total_count=628653, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.304, cer_ctc=0.053, cer=0.053, loss=4.304, time=34.13 seconds, total_count=25137, gpu_max_cached_mem_GB=10.861, [att_plot] time=1 minute and 0.38 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 14:56:19,317 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 14:56:19,320 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/56epoch.pth +[ALab02] 2023-06-11 14:56:19,320 (trainer:272) INFO: 58/70epoch started. Estimated time to finish: 11 hours, 28 minutes and 31.97 seconds +[ALab02] 2023-06-11 14:58:33,344 (trainer:721) INFO: 58epoch:train:1-551batch: iter_time=0.002, forward_time=0.090, loss_ctc=0.695, loss=0.695, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.359e-04, train_time=0.973 +[ALab02] 2023-06-11 15:00:40,191 (trainer:721) INFO: 58epoch:train:552-1102batch: iter_time=3.850e-04, forward_time=0.086, loss_ctc=0.694, loss=0.694, backward_time=0.111, optim_step_time=0.052, optim0_lr0=2.358e-04, train_time=0.921 +[ALab02] 2023-06-11 15:02:38,386 (trainer:721) INFO: 58epoch:train:1103-1653batch: iter_time=2.994e-04, forward_time=0.079, loss_ctc=0.684, loss=0.684, backward_time=0.104, optim_step_time=0.050, optim0_lr0=2.357e-04, train_time=0.857 +[ALab02] 2023-06-11 15:04:44,705 (trainer:721) INFO: 58epoch:train:1654-2204batch: iter_time=3.571e-04, forward_time=0.086, loss_ctc=0.728, loss=0.728, backward_time=0.110, optim_step_time=0.054, optim0_lr0=2.356e-04, train_time=0.917 +[ALab02] 2023-06-11 15:06:40,157 (trainer:721) INFO: 58epoch:train:2205-2755batch: iter_time=3.035e-04, forward_time=0.078, loss_ctc=0.675, loss=0.675, backward_time=0.101, optim_step_time=0.047, optim0_lr0=2.355e-04, train_time=0.838 +[ALab02] 2023-06-11 15:08:34,188 (trainer:721) INFO: 58epoch:train:2756-3306batch: iter_time=3.000e-04, forward_time=0.076, loss_ctc=0.666, loss=0.666, backward_time=0.101, optim_step_time=0.047, optim0_lr0=2.354e-04, train_time=0.828 +[ALab02] 2023-06-11 15:10:37,413 (trainer:721) INFO: 58epoch:train:3307-3857batch: iter_time=3.456e-04, forward_time=0.083, loss_ctc=0.702, loss=0.702, backward_time=0.108, optim_step_time=0.052, optim0_lr0=2.353e-04, train_time=0.893 +[ALab02] 2023-06-11 15:12:37,843 (trainer:721) INFO: 58epoch:train:3858-4408batch: iter_time=3.088e-04, forward_time=0.083, loss_ctc=0.677, loss=0.677, backward_time=0.104, optim_step_time=0.049, optim0_lr0=2.352e-04, train_time=0.874 +[ALab02] 2023-06-11 15:14:37,421 (trainer:721) INFO: 58epoch:train:4409-4959batch: iter_time=3.176e-04, forward_time=0.081, loss_ctc=0.696, loss=0.696, backward_time=0.104, optim_step_time=0.051, optim0_lr0=2.351e-04, train_time=0.869 +[ALab02] 2023-06-11 15:16:29,670 (trainer:721) INFO: 58epoch:train:4960-5510batch: iter_time=2.798e-04, forward_time=0.075, loss_ctc=0.709, loss=0.709, backward_time=0.098, optim_step_time=0.047, optim0_lr0=2.350e-04, train_time=0.814 +[ALab02] 2023-06-11 15:18:29,350 (trainer:721) INFO: 58epoch:train:5511-6061batch: iter_time=2.912e-04, forward_time=0.081, loss_ctc=0.701, loss=0.701, backward_time=0.105, optim_step_time=0.050, optim0_lr0=2.349e-04, train_time=0.869 +[ALab02] 2023-06-11 15:20:17,001 (trainer:721) INFO: 58epoch:train:6062-6612batch: iter_time=2.711e-04, forward_time=0.072, loss_ctc=0.765, loss=0.765, backward_time=0.094, optim_step_time=0.045, optim0_lr0=2.348e-04, train_time=0.781 +[ALab02] 2023-06-11 15:22:04,356 (trainer:721) INFO: 58epoch:train:6613-7163batch: iter_time=2.822e-04, forward_time=0.072, loss_ctc=0.723, loss=0.723, backward_time=0.094, optim_step_time=0.044, optim0_lr0=2.347e-04, train_time=0.780 +[ALab02] 2023-06-11 15:23:51,788 (trainer:721) INFO: 58epoch:train:7164-7714batch: iter_time=2.761e-04, forward_time=0.073, loss_ctc=0.715, loss=0.715, backward_time=0.093, optim_step_time=0.044, optim0_lr0=2.346e-04, train_time=0.779 +[ALab02] 2023-06-11 15:25:51,071 (trainer:721) INFO: 58epoch:train:7715-8265batch: iter_time=2.937e-04, forward_time=0.080, loss_ctc=0.726, loss=0.726, backward_time=0.105, optim_step_time=0.050, optim0_lr0=2.345e-04, train_time=0.865 +[ALab02] 2023-06-11 15:27:40,278 (trainer:721) INFO: 58epoch:train:8266-8816batch: iter_time=2.876e-04, forward_time=0.073, loss_ctc=0.678, loss=0.678, backward_time=0.095, optim_step_time=0.046, optim0_lr0=2.344e-04, train_time=0.792 +[ALab02] 2023-06-11 15:29:26,821 (trainer:721) INFO: 58epoch:train:8817-9367batch: iter_time=2.781e-04, forward_time=0.072, loss_ctc=0.739, loss=0.739, backward_time=0.093, optim_step_time=0.045, optim0_lr0=2.343e-04, train_time=0.774 +[ALab02] 2023-06-11 15:31:07,913 (trainer:721) INFO: 58epoch:train:9368-9918batch: iter_time=2.451e-04, forward_time=0.068, loss_ctc=0.739, loss=0.739, backward_time=0.087, optim_step_time=0.043, optim0_lr0=2.342e-04, train_time=0.734 +[ALab02] 2023-06-11 15:32:49,014 (trainer:721) INFO: 58epoch:train:9919-10469batch: iter_time=2.383e-04, forward_time=0.067, loss_ctc=0.715, loss=0.715, backward_time=0.089, optim_step_time=0.041, optim0_lr0=2.341e-04, train_time=0.733 +[ALab02] 2023-06-11 15:34:29,674 (trainer:721) INFO: 58epoch:train:10470-11020batch: iter_time=2.340e-04, forward_time=0.067, loss_ctc=0.725, loss=0.725, backward_time=0.088, optim_step_time=0.042, optim0_lr0=2.340e-04, train_time=0.730 +[ALab02] 2023-06-11 15:35:30,418 (trainer:338) INFO: 58epoch results: [train] iter_time=3.860e-04, forward_time=0.077, loss_ctc=0.707, loss=0.707, backward_time=0.100, optim_step_time=0.048, optim0_lr0=2.349e-04, train_time=0.831, time=38 minutes and 12.31 seconds, total_count=639682, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.366, cer_ctc=0.053, cer=0.053, loss=4.366, time=25.61 seconds, total_count=25578, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.18 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 15:35:33,210 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 15:35:33,213 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/45epoch.pth +[ALab02] 2023-06-11 15:35:33,214 (trainer:272) INFO: 59/70epoch started. Estimated time to finish: 10 hours, 32 minutes and 43.65 seconds +[ALab02] 2023-06-11 15:37:09,247 (trainer:721) INFO: 59epoch:train:1-551batch: iter_time=9.725e-04, forward_time=0.063, loss_ctc=0.694, loss=0.694, backward_time=0.083, optim_step_time=0.039, optim0_lr0=2.339e-04, train_time=0.697 +[ALab02] 2023-06-11 15:38:41,006 (trainer:721) INFO: 59epoch:train:552-1102batch: iter_time=2.122e-04, forward_time=0.059, loss_ctc=0.727, loss=0.727, backward_time=0.081, optim_step_time=0.038, optim0_lr0=2.338e-04, train_time=0.666 +[ALab02] 2023-06-11 15:40:09,255 (trainer:721) INFO: 59epoch:train:1103-1653batch: iter_time=1.923e-04, forward_time=0.056, loss_ctc=0.702, loss=0.702, backward_time=0.078, optim_step_time=0.036, optim0_lr0=2.337e-04, train_time=0.640 +[ALab02] 2023-06-11 15:41:45,824 (trainer:721) INFO: 59epoch:train:1654-2204batch: iter_time=2.162e-04, forward_time=0.062, loss_ctc=0.695, loss=0.695, backward_time=0.086, optim_step_time=0.040, optim0_lr0=2.336e-04, train_time=0.701 +[ALab02] 2023-06-11 15:43:12,566 (trainer:721) INFO: 59epoch:train:2205-2755batch: iter_time=1.907e-04, forward_time=0.055, loss_ctc=0.717, loss=0.717, backward_time=0.076, optim_step_time=0.036, optim0_lr0=2.335e-04, train_time=0.630 +[ALab02] 2023-06-11 15:44:38,458 (trainer:721) INFO: 59epoch:train:2756-3306batch: iter_time=1.901e-04, forward_time=0.054, loss_ctc=0.681, loss=0.681, backward_time=0.076, optim_step_time=0.036, optim0_lr0=2.334e-04, train_time=0.623 +[ALab02] 2023-06-11 15:46:01,372 (trainer:721) INFO: 59epoch:train:3307-3857batch: iter_time=1.613e-04, forward_time=0.053, loss_ctc=0.717, loss=0.717, backward_time=0.072, optim_step_time=0.035, optim0_lr0=2.333e-04, train_time=0.602 +[ALab02] 2023-06-11 15:48:16,269 (trainer:721) INFO: 59epoch:train:3858-4408batch: iter_time=3.064e-04, forward_time=0.092, loss_ctc=0.731, loss=0.731, backward_time=0.118, optim_step_time=0.058, optim0_lr0=2.332e-04, train_time=0.978 +[ALab02] 2023-06-11 15:50:40,352 (trainer:721) INFO: 59epoch:train:4409-4959batch: iter_time=3.512e-04, forward_time=0.100, loss_ctc=0.752, loss=0.752, backward_time=0.125, optim_step_time=0.063, optim0_lr0=2.331e-04, train_time=1.045 +[ALab02] 2023-06-11 15:52:50,837 (trainer:721) INFO: 59epoch:train:4960-5510batch: iter_time=3.224e-04, forward_time=0.089, loss_ctc=0.698, loss=0.698, backward_time=0.115, optim_step_time=0.054, optim0_lr0=2.330e-04, train_time=0.948 +[ALab02] 2023-06-11 15:55:12,157 (trainer:721) INFO: 59epoch:train:5511-6061batch: iter_time=3.460e-04, forward_time=0.097, loss_ctc=0.695, loss=0.695, backward_time=0.124, optim_step_time=0.060, optim0_lr0=2.329e-04, train_time=1.025 +[ALab02] 2023-06-11 15:57:21,673 (trainer:721) INFO: 59epoch:train:6062-6612batch: iter_time=3.290e-04, forward_time=0.091, loss_ctc=0.739, loss=0.739, backward_time=0.111, optim_step_time=0.054, optim0_lr0=2.328e-04, train_time=0.940 +[ALab02] 2023-06-11 15:59:26,991 (trainer:721) INFO: 59epoch:train:6613-7163batch: iter_time=3.037e-04, forward_time=0.085, loss_ctc=0.731, loss=0.731, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.327e-04, train_time=0.910 +[ALab02] 2023-06-11 16:01:45,404 (trainer:721) INFO: 59epoch:train:7164-7714batch: iter_time=3.529e-04, forward_time=0.095, loss_ctc=0.762, loss=0.762, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.326e-04, train_time=1.004 +[ALab02] 2023-06-11 16:03:53,762 (trainer:721) INFO: 59epoch:train:7715-8265batch: iter_time=3.261e-04, forward_time=0.089, loss_ctc=0.716, loss=0.716, backward_time=0.111, optim_step_time=0.054, optim0_lr0=2.325e-04, train_time=0.931 +[ALab02] 2023-06-11 16:05:59,585 (trainer:721) INFO: 59epoch:train:8266-8816batch: iter_time=3.089e-04, forward_time=0.087, loss_ctc=0.756, loss=0.756, backward_time=0.109, optim_step_time=0.053, optim0_lr0=2.324e-04, train_time=0.913 +[ALab02] 2023-06-11 16:08:01,225 (trainer:721) INFO: 59epoch:train:8817-9367batch: iter_time=3.087e-04, forward_time=0.084, loss_ctc=0.749, loss=0.749, backward_time=0.105, optim_step_time=0.050, optim0_lr0=2.323e-04, train_time=0.884 +[ALab02] 2023-06-11 16:10:22,695 (trainer:721) INFO: 59epoch:train:9368-9918batch: iter_time=3.919e-04, forward_time=0.096, loss_ctc=0.717, loss=0.717, backward_time=0.126, optim_step_time=0.058, optim0_lr0=2.322e-04, train_time=1.025 +[ALab02] 2023-06-11 16:12:34,745 (trainer:721) INFO: 59epoch:train:9919-10469batch: iter_time=3.391e-04, forward_time=0.089, loss_ctc=0.723, loss=0.723, backward_time=0.117, optim_step_time=0.055, optim0_lr0=2.321e-04, train_time=0.959 +[ALab02] 2023-06-11 16:14:48,204 (trainer:721) INFO: 59epoch:train:10470-11020batch: iter_time=3.220e-04, forward_time=0.091, loss_ctc=0.708, loss=0.708, backward_time=0.117, optim_step_time=0.055, optim0_lr0=2.320e-04, train_time=0.968 +[ALab02] 2023-06-11 16:16:01,845 (trainer:338) INFO: 59epoch results: [train] iter_time=3.221e-04, forward_time=0.079, loss_ctc=0.720, loss=0.720, backward_time=0.103, optim_step_time=0.049, optim0_lr0=2.329e-04, train_time=0.854, time=39 minutes and 17.45 seconds, total_count=650711, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.457, cer_ctc=0.054, cer=0.054, loss=4.457, time=30.48 seconds, total_count=26019, gpu_max_cached_mem_GB=10.861, [att_plot] time=40.7 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 16:16:05,631 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 16:16:05,633 (trainer:272) INFO: 60/70epoch started. Estimated time to finish: 9 hours, 37 minutes and 43.69 seconds +[ALab02] 2023-06-11 16:18:27,821 (trainer:721) INFO: 60epoch:train:1-551batch: iter_time=0.002, forward_time=0.096, loss_ctc=0.744, loss=0.744, backward_time=0.124, optim_step_time=0.058, optim0_lr0=2.319e-04, train_time=1.033 +[ALab02] 2023-06-11 16:20:40,377 (trainer:721) INFO: 60epoch:train:552-1102batch: iter_time=3.319e-04, forward_time=0.092, loss_ctc=0.725, loss=0.725, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.318e-04, train_time=0.962 +[ALab02] 2023-06-11 16:22:46,297 (trainer:721) INFO: 60epoch:train:1103-1653batch: iter_time=3.169e-04, forward_time=0.087, loss_ctc=0.713, loss=0.713, backward_time=0.109, optim_step_time=0.052, optim0_lr0=2.317e-04, train_time=0.914 +[ALab02] 2023-06-11 16:25:03,844 (trainer:721) INFO: 60epoch:train:1654-2204batch: iter_time=3.455e-04, forward_time=0.094, loss_ctc=0.718, loss=0.718, backward_time=0.121, optim_step_time=0.058, optim0_lr0=2.316e-04, train_time=0.998 +[ALab02] 2023-06-11 16:27:14,531 (trainer:721) INFO: 60epoch:train:2205-2755batch: iter_time=3.382e-04, forward_time=0.090, loss_ctc=0.719, loss=0.719, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.315e-04, train_time=0.949 +[ALab02] 2023-06-11 16:29:14,792 (trainer:721) INFO: 60epoch:train:2756-3306batch: iter_time=3.003e-04, forward_time=0.082, loss_ctc=0.730, loss=0.730, backward_time=0.105, optim_step_time=0.049, optim0_lr0=2.314e-04, train_time=0.873 +[ALab02] 2023-06-11 16:31:37,887 (trainer:721) INFO: 60epoch:train:3307-3857batch: iter_time=3.462e-04, forward_time=0.100, loss_ctc=0.739, loss=0.739, backward_time=0.124, optim_step_time=0.061, optim0_lr0=2.313e-04, train_time=1.037 +[ALab02] 2023-06-11 16:33:55,697 (trainer:721) INFO: 60epoch:train:3858-4408batch: iter_time=3.440e-04, forward_time=0.096, loss_ctc=0.729, loss=0.729, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.312e-04, train_time=1.000 +[ALab02] 2023-06-11 16:36:00,441 (trainer:721) INFO: 60epoch:train:4409-4959batch: iter_time=3.028e-04, forward_time=0.087, loss_ctc=0.703, loss=0.703, backward_time=0.108, optim_step_time=0.052, optim0_lr0=2.311e-04, train_time=0.906 +[ALab02] 2023-06-11 16:38:22,360 (trainer:721) INFO: 60epoch:train:4960-5510batch: iter_time=3.490e-04, forward_time=0.099, loss_ctc=0.750, loss=0.750, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.310e-04, train_time=1.029 +[ALab02] 2023-06-11 16:40:37,312 (trainer:721) INFO: 60epoch:train:5511-6061batch: iter_time=3.316e-04, forward_time=0.093, loss_ctc=0.725, loss=0.725, backward_time=0.117, optim_step_time=0.055, optim0_lr0=2.309e-04, train_time=0.979 +[ALab02] 2023-06-11 16:42:40,755 (trainer:721) INFO: 60epoch:train:6062-6612batch: iter_time=2.980e-04, forward_time=0.084, loss_ctc=0.769, loss=0.769, backward_time=0.108, optim_step_time=0.050, optim0_lr0=2.308e-04, train_time=0.896 +[ALab02] 2023-06-11 16:44:48,404 (trainer:721) INFO: 60epoch:train:6613-7163batch: iter_time=3.025e-04, forward_time=0.087, loss_ctc=0.717, loss=0.717, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.307e-04, train_time=0.927 +[ALab02] 2023-06-11 16:47:05,374 (trainer:721) INFO: 60epoch:train:7164-7714batch: iter_time=3.226e-04, forward_time=0.094, loss_ctc=0.757, loss=0.757, backward_time=0.120, optim_step_time=0.058, optim0_lr0=2.306e-04, train_time=0.994 +[ALab02] 2023-06-11 16:49:10,412 (trainer:721) INFO: 60epoch:train:7715-8265batch: iter_time=3.047e-04, forward_time=0.085, loss_ctc=0.726, loss=0.726, backward_time=0.110, optim_step_time=0.051, optim0_lr0=2.305e-04, train_time=0.907 +[ALab02] 2023-06-11 16:51:07,639 (trainer:721) INFO: 60epoch:train:8266-8816batch: iter_time=2.919e-04, forward_time=0.080, loss_ctc=0.738, loss=0.738, backward_time=0.102, optim_step_time=0.048, optim0_lr0=2.304e-04, train_time=0.850 +[ALab02] 2023-06-11 16:53:12,100 (trainer:721) INFO: 60epoch:train:8817-9367batch: iter_time=2.942e-04, forward_time=0.086, loss_ctc=0.747, loss=0.747, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.303e-04, train_time=0.904 +[ALab02] 2023-06-11 16:55:07,875 (trainer:721) INFO: 60epoch:train:9368-9918batch: iter_time=2.867e-04, forward_time=0.077, loss_ctc=0.760, loss=0.760, backward_time=0.103, optim_step_time=0.045, optim0_lr0=2.302e-04, train_time=0.840 +[ALab02] 2023-06-11 16:56:59,921 (trainer:721) INFO: 60epoch:train:9919-10469batch: iter_time=2.680e-04, forward_time=0.075, loss_ctc=0.739, loss=0.739, backward_time=0.098, optim_step_time=0.044, optim0_lr0=2.301e-04, train_time=0.813 +[ALab02] 2023-06-11 16:58:49,792 (trainer:721) INFO: 60epoch:train:10470-11020batch: iter_time=2.658e-04, forward_time=0.074, loss_ctc=0.700, loss=0.700, backward_time=0.097, optim_step_time=0.044, optim0_lr0=2.300e-04, train_time=0.797 +[ALab02] 2023-06-11 16:59:52,997 (trainer:338) INFO: 60epoch results: [train] iter_time=4.066e-04, forward_time=0.088, loss_ctc=0.732, loss=0.732, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.310e-04, train_time=0.930, time=42 minutes and 46.56 seconds, total_count=661740, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.525, cer_ctc=0.054, cer=0.054, loss=4.525, time=26.27 seconds, total_count=26460, gpu_max_cached_mem_GB=10.861, [att_plot] time=34.53 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 16:59:55,694 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 16:59:55,695 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/59epoch.pth +[ALab02] 2023-06-11 16:59:55,696 (trainer:272) INFO: 61/70epoch started. Estimated time to finish: 8 hours, 43 minutes and 45.58 seconds +[ALab02] 2023-06-11 17:01:35,513 (trainer:721) INFO: 61epoch:train:1-551batch: iter_time=0.001, forward_time=0.065, loss_ctc=0.717, loss=0.717, backward_time=0.087, optim_step_time=0.040, optim0_lr0=2.299e-04, train_time=0.725 +[ALab02] 2023-06-11 17:03:09,199 (trainer:721) INFO: 61epoch:train:552-1102batch: iter_time=2.173e-04, forward_time=0.060, loss_ctc=0.716, loss=0.716, backward_time=0.083, optim_step_time=0.038, optim0_lr0=2.298e-04, train_time=0.680 +[ALab02] 2023-06-11 17:04:42,525 (trainer:721) INFO: 61epoch:train:1103-1653batch: iter_time=2.394e-04, forward_time=0.060, loss_ctc=0.723, loss=0.723, backward_time=0.082, optim_step_time=0.040, optim0_lr0=2.298e-04, train_time=0.676 +[ALab02] 2023-06-11 17:06:17,035 (trainer:721) INFO: 61epoch:train:1654-2204batch: iter_time=2.133e-04, forward_time=0.061, loss_ctc=0.696, loss=0.696, backward_time=0.083, optim_step_time=0.038, optim0_lr0=2.297e-04, train_time=0.686 +[ALab02] 2023-06-11 17:07:49,614 (trainer:721) INFO: 61epoch:train:2205-2755batch: iter_time=2.116e-04, forward_time=0.058, loss_ctc=0.715, loss=0.715, backward_time=0.083, optim_step_time=0.038, optim0_lr0=2.296e-04, train_time=0.672 +[ALab02] 2023-06-11 17:09:44,499 (trainer:721) INFO: 61epoch:train:2756-3306batch: iter_time=2.764e-04, forward_time=0.077, loss_ctc=0.708, loss=0.708, backward_time=0.101, optim_step_time=0.045, optim0_lr0=2.295e-04, train_time=0.833 +[ALab02] 2023-06-11 17:11:53,650 (trainer:721) INFO: 61epoch:train:3307-3857batch: iter_time=3.142e-04, forward_time=0.087, loss_ctc=0.676, loss=0.676, backward_time=0.115, optim_step_time=0.052, optim0_lr0=2.294e-04, train_time=0.937 +[ALab02] 2023-06-11 17:13:59,613 (trainer:721) INFO: 61epoch:train:3858-4408batch: iter_time=3.158e-04, forward_time=0.086, loss_ctc=0.720, loss=0.720, backward_time=0.110, optim_step_time=0.052, optim0_lr0=2.293e-04, train_time=0.914 +[ALab02] 2023-06-11 17:16:06,848 (trainer:721) INFO: 61epoch:train:4409-4959batch: iter_time=3.125e-04, forward_time=0.087, loss_ctc=0.661, loss=0.661, backward_time=0.111, optim_step_time=0.053, optim0_lr0=2.292e-04, train_time=0.924 +[ALab02] 2023-06-11 17:18:01,718 (trainer:721) INFO: 61epoch:train:4960-5510batch: iter_time=2.787e-04, forward_time=0.078, loss_ctc=0.697, loss=0.697, backward_time=0.101, optim_step_time=0.046, optim0_lr0=2.291e-04, train_time=0.832 +[ALab02] 2023-06-11 17:20:11,221 (trainer:721) INFO: 61epoch:train:5511-6061batch: iter_time=3.285e-04, forward_time=0.088, loss_ctc=0.683, loss=0.683, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.290e-04, train_time=0.940 +[ALab02] 2023-06-11 17:22:25,499 (trainer:721) INFO: 61epoch:train:6062-6612batch: iter_time=3.684e-04, forward_time=0.092, loss_ctc=0.687, loss=0.687, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.289e-04, train_time=0.975 +[ALab02] 2023-06-11 17:24:33,340 (trainer:721) INFO: 61epoch:train:6613-7163batch: iter_time=3.052e-04, forward_time=0.087, loss_ctc=0.721, loss=0.721, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.288e-04, train_time=0.927 +[ALab02] 2023-06-11 17:26:32,585 (trainer:721) INFO: 61epoch:train:7164-7714batch: iter_time=3.078e-04, forward_time=0.081, loss_ctc=0.673, loss=0.673, backward_time=0.104, optim_step_time=0.048, optim0_lr0=2.287e-04, train_time=0.867 +[ALab02] 2023-06-11 17:28:41,620 (trainer:721) INFO: 61epoch:train:7715-8265batch: iter_time=3.684e-04, forward_time=0.089, loss_ctc=0.693, loss=0.693, backward_time=0.111, optim_step_time=0.055, optim0_lr0=2.286e-04, train_time=0.935 +[ALab02] 2023-06-11 17:30:50,958 (trainer:721) INFO: 61epoch:train:8266-8816batch: iter_time=3.190e-04, forward_time=0.088, loss_ctc=0.653, loss=0.653, backward_time=0.114, optim_step_time=0.052, optim0_lr0=2.285e-04, train_time=0.939 +[ALab02] 2023-06-11 17:32:55,871 (trainer:721) INFO: 61epoch:train:8817-9367batch: iter_time=3.046e-04, forward_time=0.085, loss_ctc=0.695, loss=0.695, backward_time=0.109, optim_step_time=0.052, optim0_lr0=2.284e-04, train_time=0.907 +[ALab02] 2023-06-11 17:34:53,534 (trainer:721) INFO: 61epoch:train:9368-9918batch: iter_time=3.479e-04, forward_time=0.080, loss_ctc=0.680, loss=0.680, backward_time=0.103, optim_step_time=0.049, optim0_lr0=2.283e-04, train_time=0.853 +[ALab02] 2023-06-11 17:37:12,339 (trainer:721) INFO: 61epoch:train:9919-10469batch: iter_time=3.571e-04, forward_time=0.096, loss_ctc=0.669, loss=0.669, backward_time=0.120, optim_step_time=0.060, optim0_lr0=2.282e-04, train_time=1.008 +[ALab02] 2023-06-11 17:39:23,532 (trainer:721) INFO: 61epoch:train:10470-11020batch: iter_time=3.214e-04, forward_time=0.091, loss_ctc=0.698, loss=0.698, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.281e-04, train_time=0.952 +[ALab02] 2023-06-11 17:40:39,608 (trainer:338) INFO: 61epoch results: [train] iter_time=3.436e-04, forward_time=0.080, loss_ctc=0.694, loss=0.694, backward_time=0.104, optim_step_time=0.049, optim0_lr0=2.290e-04, train_time=0.859, time=39 minutes and 30.63 seconds, total_count=672769, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.503, cer_ctc=0.052, cer=0.052, loss=4.503, time=32.26 seconds, total_count=26901, gpu_max_cached_mem_GB=10.861, [att_plot] time=41.02 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 17:40:42,802 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-11 17:40:42,810 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/55epoch.pth, exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/60epoch.pth +[ALab02] 2023-06-11 17:40:42,810 (trainer:272) INFO: 62/70epoch started. Estimated time to finish: 7 hours, 49 minutes and 40.41 seconds +[ALab02] 2023-06-11 17:42:54,974 (trainer:721) INFO: 62epoch:train:1-551batch: iter_time=0.002, forward_time=0.089, loss_ctc=0.682, loss=0.682, backward_time=0.116, optim_step_time=0.053, optim0_lr0=2.281e-04, train_time=0.959 +[ALab02] 2023-06-11 17:45:05,967 (trainer:721) INFO: 62epoch:train:552-1102batch: iter_time=3.195e-04, forward_time=0.090, loss_ctc=0.669, loss=0.669, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.280e-04, train_time=0.951 +[ALab02] 2023-06-11 17:47:15,550 (trainer:721) INFO: 62epoch:train:1103-1653batch: iter_time=3.221e-04, forward_time=0.089, loss_ctc=0.647, loss=0.647, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.279e-04, train_time=0.940 +[ALab02] 2023-06-11 17:49:09,067 (trainer:721) INFO: 62epoch:train:1654-2204batch: iter_time=2.829e-04, forward_time=0.077, loss_ctc=0.662, loss=0.662, backward_time=0.099, optim_step_time=0.046, optim0_lr0=2.278e-04, train_time=0.824 +[ALab02] 2023-06-11 17:51:29,844 (trainer:721) INFO: 62epoch:train:2205-2755batch: iter_time=3.712e-04, forward_time=0.096, loss_ctc=0.657, loss=0.657, backward_time=0.125, optim_step_time=0.058, optim0_lr0=2.277e-04, train_time=1.022 +[ALab02] 2023-06-11 17:53:38,350 (trainer:721) INFO: 62epoch:train:2756-3306batch: iter_time=3.134e-04, forward_time=0.088, loss_ctc=0.649, loss=0.649, backward_time=0.113, optim_step_time=0.052, optim0_lr0=2.276e-04, train_time=0.933 +[ALab02] 2023-06-11 17:55:38,439 (trainer:721) INFO: 62epoch:train:3307-3857batch: iter_time=2.945e-04, forward_time=0.082, loss_ctc=0.656, loss=0.656, backward_time=0.104, optim_step_time=0.050, optim0_lr0=2.275e-04, train_time=0.871 +[ALab02] 2023-06-11 17:57:40,387 (trainer:721) INFO: 62epoch:train:3858-4408batch: iter_time=3.292e-04, forward_time=0.083, loss_ctc=0.651, loss=0.651, backward_time=0.107, optim_step_time=0.050, optim0_lr0=2.274e-04, train_time=0.885 +[ALab02] 2023-06-11 17:59:59,588 (trainer:721) INFO: 62epoch:train:4409-4959batch: iter_time=3.351e-04, forward_time=0.097, loss_ctc=0.663, loss=0.663, backward_time=0.121, optim_step_time=0.059, optim0_lr0=2.273e-04, train_time=1.009 +[ALab02] 2023-06-11 18:02:07,261 (trainer:721) INFO: 62epoch:train:4960-5510batch: iter_time=3.098e-04, forward_time=0.087, loss_ctc=0.671, loss=0.671, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.272e-04, train_time=0.928 +[ALab02] 2023-06-11 18:04:10,687 (trainer:721) INFO: 62epoch:train:5511-6061batch: iter_time=3.030e-04, forward_time=0.085, loss_ctc=0.659, loss=0.659, backward_time=0.108, optim_step_time=0.050, optim0_lr0=2.271e-04, train_time=0.895 +[ALab02] 2023-06-11 18:06:02,736 (trainer:721) INFO: 62epoch:train:6062-6612batch: iter_time=2.736e-04, forward_time=0.076, loss_ctc=0.725, loss=0.725, backward_time=0.098, optim_step_time=0.045, optim0_lr0=2.270e-04, train_time=0.813 +[ALab02] 2023-06-11 18:08:20,305 (trainer:721) INFO: 62epoch:train:6613-7163batch: iter_time=3.230e-04, forward_time=0.093, loss_ctc=0.687, loss=0.687, backward_time=0.123, optim_step_time=0.055, optim0_lr0=2.269e-04, train_time=0.999 +[ALab02] 2023-06-11 18:10:25,741 (trainer:721) INFO: 62epoch:train:7164-7714batch: iter_time=3.131e-04, forward_time=0.086, loss_ctc=0.661, loss=0.661, backward_time=0.109, optim_step_time=0.051, optim0_lr0=2.268e-04, train_time=0.910 +[ALab02] 2023-06-11 18:12:20,474 (trainer:721) INFO: 62epoch:train:7715-8265batch: iter_time=2.738e-04, forward_time=0.078, loss_ctc=0.679, loss=0.679, backward_time=0.100, optim_step_time=0.045, optim0_lr0=2.268e-04, train_time=0.833 +[ALab02] 2023-06-11 18:14:22,157 (trainer:721) INFO: 62epoch:train:8266-8816batch: iter_time=2.829e-04, forward_time=0.084, loss_ctc=0.694, loss=0.694, backward_time=0.106, optim_step_time=0.050, optim0_lr0=2.267e-04, train_time=0.882 +[ALab02] 2023-06-11 18:16:17,310 (trainer:721) INFO: 62epoch:train:8817-9367batch: iter_time=2.756e-04, forward_time=0.078, loss_ctc=0.671, loss=0.671, backward_time=0.101, optim_step_time=0.046, optim0_lr0=2.266e-04, train_time=0.836 +[ALab02] 2023-06-11 18:18:04,524 (trainer:721) INFO: 62epoch:train:9368-9918batch: iter_time=2.550e-04, forward_time=0.072, loss_ctc=0.692, loss=0.692, backward_time=0.094, optim_step_time=0.043, optim0_lr0=2.265e-04, train_time=0.778 +[ALab02] 2023-06-11 18:19:36,592 (trainer:721) INFO: 62epoch:train:9919-10469batch: iter_time=1.877e-04, forward_time=0.060, loss_ctc=0.671, loss=0.671, backward_time=0.080, optim_step_time=0.038, optim0_lr0=2.264e-04, train_time=0.668 +[ALab02] 2023-06-11 18:21:25,257 (trainer:721) INFO: 62epoch:train:10470-11020batch: iter_time=2.390e-04, forward_time=0.070, loss_ctc=0.651, loss=0.651, backward_time=0.097, optim_step_time=0.044, optim0_lr0=2.263e-04, train_time=0.788 +[ALab02] 2023-06-11 18:22:26,317 (trainer:338) INFO: 62epoch results: [train] iter_time=3.581e-04, forward_time=0.083, loss_ctc=0.670, loss=0.670, backward_time=0.107, optim_step_time=0.050, optim0_lr0=2.272e-04, train_time=0.886, time=40 minutes and 44.48 seconds, total_count=683798, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.413, cer_ctc=0.053, cer=0.053, loss=4.413, time=25.06 seconds, total_count=27342, gpu_max_cached_mem_GB=10.861, [att_plot] time=33.97 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 18:22:28,935 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 18:22:28,936 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/42epoch.pth +[ALab02] 2023-06-11 18:22:28,937 (trainer:272) INFO: 63/70epoch started. Estimated time to finish: 6 hours, 56 minutes and 8.61 seconds +[ALab02] 2023-06-11 18:24:06,199 (trainer:721) INFO: 63epoch:train:1-551batch: iter_time=8.659e-04, forward_time=0.063, loss_ctc=0.670, loss=0.670, backward_time=0.086, optim_step_time=0.039, optim0_lr0=2.262e-04, train_time=0.706 +[ALab02] 2023-06-11 18:25:31,712 (trainer:721) INFO: 63epoch:train:552-1102batch: iter_time=1.960e-04, forward_time=0.055, loss_ctc=0.659, loss=0.659, backward_time=0.075, optim_step_time=0.036, optim0_lr0=2.261e-04, train_time=0.621 +[ALab02] 2023-06-11 18:27:26,175 (trainer:721) INFO: 63epoch:train:1103-1653batch: iter_time=2.707e-04, forward_time=0.076, loss_ctc=0.616, loss=0.616, backward_time=0.102, optim_step_time=0.047, optim0_lr0=2.260e-04, train_time=0.828 +[ALab02] 2023-06-11 18:29:58,070 (trainer:721) INFO: 63epoch:train:1654-2204batch: iter_time=3.656e-04, forward_time=0.105, loss_ctc=0.637, loss=0.637, backward_time=0.134, optim_step_time=0.062, optim0_lr0=2.259e-04, train_time=1.103 +[ALab02] 2023-06-11 18:32:08,399 (trainer:721) INFO: 63epoch:train:2205-2755batch: iter_time=3.322e-04, forward_time=0.090, loss_ctc=0.680, loss=0.680, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.258e-04, train_time=0.947 +[ALab02] 2023-06-11 18:34:18,052 (trainer:721) INFO: 63epoch:train:2756-3306batch: iter_time=3.217e-04, forward_time=0.088, loss_ctc=0.689, loss=0.689, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.258e-04, train_time=0.941 +[ALab02] 2023-06-11 18:36:40,824 (trainer:721) INFO: 63epoch:train:3307-3857batch: iter_time=3.660e-04, forward_time=0.098, loss_ctc=0.660, loss=0.660, backward_time=0.125, optim_step_time=0.060, optim0_lr0=2.257e-04, train_time=1.035 +[ALab02] 2023-06-11 18:38:55,427 (trainer:721) INFO: 63epoch:train:3858-4408batch: iter_time=3.212e-04, forward_time=0.093, loss_ctc=0.712, loss=0.712, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.256e-04, train_time=0.977 +[ALab02] 2023-06-11 18:41:07,788 (trainer:721) INFO: 63epoch:train:4409-4959batch: iter_time=3.164e-04, forward_time=0.090, loss_ctc=0.662, loss=0.662, backward_time=0.117, optim_step_time=0.054, optim0_lr0=2.255e-04, train_time=0.961 +[ALab02] 2023-06-11 18:43:08,178 (trainer:721) INFO: 63epoch:train:4960-5510batch: iter_time=3.369e-04, forward_time=0.082, loss_ctc=0.659, loss=0.659, backward_time=0.105, optim_step_time=0.050, optim0_lr0=2.254e-04, train_time=0.872 +[ALab02] 2023-06-11 18:45:27,291 (trainer:721) INFO: 63epoch:train:5511-6061batch: iter_time=3.627e-04, forward_time=0.094, loss_ctc=0.647, loss=0.647, backward_time=0.124, optim_step_time=0.056, optim0_lr0=2.253e-04, train_time=1.011 +[ALab02] 2023-06-11 18:47:41,777 (trainer:721) INFO: 63epoch:train:6062-6612batch: iter_time=3.341e-04, forward_time=0.091, loss_ctc=0.631, loss=0.631, backward_time=0.119, optim_step_time=0.055, optim0_lr0=2.252e-04, train_time=0.975 +[ALab02] 2023-06-11 18:49:41,405 (trainer:721) INFO: 63epoch:train:6613-7163batch: iter_time=3.040e-04, forward_time=0.081, loss_ctc=0.643, loss=0.643, backward_time=0.105, optim_step_time=0.048, optim0_lr0=2.251e-04, train_time=0.869 +[ALab02] 2023-06-11 18:51:58,589 (trainer:721) INFO: 63epoch:train:7164-7714batch: iter_time=3.167e-04, forward_time=0.095, loss_ctc=0.657, loss=0.657, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.250e-04, train_time=0.994 +[ALab02] 2023-06-11 18:54:12,555 (trainer:721) INFO: 63epoch:train:7715-8265batch: iter_time=3.207e-04, forward_time=0.091, loss_ctc=0.634, loss=0.634, backward_time=0.119, optim_step_time=0.054, optim0_lr0=2.249e-04, train_time=0.972 +[ALab02] 2023-06-11 18:56:18,479 (trainer:721) INFO: 63epoch:train:8266-8816batch: iter_time=3.227e-04, forward_time=0.087, loss_ctc=0.646, loss=0.646, backward_time=0.109, optim_step_time=0.052, optim0_lr0=2.248e-04, train_time=0.914 +[ALab02] 2023-06-11 18:58:30,474 (trainer:721) INFO: 63epoch:train:8817-9367batch: iter_time=3.497e-04, forward_time=0.090, loss_ctc=0.669, loss=0.669, backward_time=0.116, optim_step_time=0.054, optim0_lr0=2.248e-04, train_time=0.958 +[ALab02] 2023-06-11 19:00:43,419 (trainer:721) INFO: 63epoch:train:9368-9918batch: iter_time=3.322e-04, forward_time=0.092, loss_ctc=0.634, loss=0.634, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.247e-04, train_time=0.964 +[ALab02] 2023-06-11 19:02:57,007 (trainer:721) INFO: 63epoch:train:9919-10469batch: iter_time=3.311e-04, forward_time=0.091, loss_ctc=0.626, loss=0.626, backward_time=0.118, optim_step_time=0.054, optim0_lr0=2.246e-04, train_time=0.970 +[ALab02] 2023-06-11 19:04:56,489 (trainer:721) INFO: 63epoch:train:10470-11020batch: iter_time=3.163e-04, forward_time=0.080, loss_ctc=0.636, loss=0.636, backward_time=0.106, optim_step_time=0.048, optim0_lr0=2.245e-04, train_time=0.867 +[ALab02] 2023-06-11 19:06:28,076 (trainer:338) INFO: 63epoch results: [train] iter_time=3.491e-04, forward_time=0.087, loss_ctc=0.653, loss=0.653, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.253e-04, train_time=0.925, time=42 minutes and 31.36 seconds, total_count=694827, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.314, cer_ctc=0.053, cer=0.053, loss=4.314, time=41.42 seconds, total_count=27783, gpu_max_cached_mem_GB=10.861, [att_plot] time=46.36 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 19:06:31,630 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 19:06:31,634 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/54epoch.pth +[ALab02] 2023-06-11 19:06:31,634 (trainer:272) INFO: 64/70epoch started. Estimated time to finish: 6 hours, 3 minutes and 14.38 seconds +[ALab02] 2023-06-11 19:08:44,946 (trainer:721) INFO: 64epoch:train:1-551batch: iter_time=0.002, forward_time=0.092, loss_ctc=0.648, loss=0.648, backward_time=0.114, optim_step_time=0.056, optim0_lr0=2.244e-04, train_time=0.968 +[ALab02] 2023-06-11 19:10:51,882 (trainer:721) INFO: 64epoch:train:552-1102batch: iter_time=3.382e-04, forward_time=0.086, loss_ctc=0.643, loss=0.643, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.243e-04, train_time=0.921 +[ALab02] 2023-06-11 19:12:53,456 (trainer:721) INFO: 64epoch:train:1103-1653batch: iter_time=2.980e-04, forward_time=0.083, loss_ctc=0.637, loss=0.637, backward_time=0.107, optim_step_time=0.050, optim0_lr0=2.242e-04, train_time=0.882 +[ALab02] 2023-06-11 19:15:10,160 (trainer:721) INFO: 64epoch:train:1654-2204batch: iter_time=3.265e-04, forward_time=0.093, loss_ctc=0.639, loss=0.639, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.241e-04, train_time=0.992 +[ALab02] 2023-06-11 19:17:24,703 (trainer:721) INFO: 64epoch:train:2205-2755batch: iter_time=3.222e-04, forward_time=0.092, loss_ctc=0.617, loss=0.617, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.240e-04, train_time=0.977 +[ALab02] 2023-06-11 19:19:35,786 (trainer:721) INFO: 64epoch:train:2756-3306batch: iter_time=3.090e-04, forward_time=0.090, loss_ctc=0.640, loss=0.640, backward_time=0.115, optim_step_time=0.053, optim0_lr0=2.240e-04, train_time=0.952 +[ALab02] 2023-06-11 19:21:34,136 (trainer:721) INFO: 64epoch:train:3307-3857batch: iter_time=2.937e-04, forward_time=0.080, loss_ctc=0.630, loss=0.630, backward_time=0.104, optim_step_time=0.048, optim0_lr0=2.239e-04, train_time=0.858 +[ALab02] 2023-06-11 19:23:53,312 (trainer:721) INFO: 64epoch:train:3858-4408batch: iter_time=4.527e-04, forward_time=0.097, loss_ctc=0.639, loss=0.639, backward_time=0.121, optim_step_time=0.057, optim0_lr0=2.238e-04, train_time=1.010 +[ALab02] 2023-06-11 19:26:02,056 (trainer:721) INFO: 64epoch:train:4409-4959batch: iter_time=3.017e-04, forward_time=0.088, loss_ctc=0.663, loss=0.663, backward_time=0.113, optim_step_time=0.053, optim0_lr0=2.237e-04, train_time=0.934 +[ALab02] 2023-06-11 19:28:00,413 (trainer:721) INFO: 64epoch:train:4960-5510batch: iter_time=2.835e-04, forward_time=0.079, loss_ctc=0.630, loss=0.630, backward_time=0.105, optim_step_time=0.047, optim0_lr0=2.236e-04, train_time=0.860 +[ALab02] 2023-06-11 19:30:02,913 (trainer:721) INFO: 64epoch:train:5511-6061batch: iter_time=3.069e-04, forward_time=0.083, loss_ctc=0.662, loss=0.662, backward_time=0.108, optim_step_time=0.049, optim0_lr0=2.235e-04, train_time=0.888 +[ALab02] 2023-06-11 19:32:05,143 (trainer:721) INFO: 64epoch:train:6062-6612batch: iter_time=3.027e-04, forward_time=0.084, loss_ctc=0.621, loss=0.621, backward_time=0.107, optim_step_time=0.050, optim0_lr0=2.234e-04, train_time=0.887 +[ALab02] 2023-06-11 19:34:03,290 (trainer:721) INFO: 64epoch:train:6613-7163batch: iter_time=2.811e-04, forward_time=0.082, loss_ctc=0.634, loss=0.634, backward_time=0.101, optim_step_time=0.049, optim0_lr0=2.233e-04, train_time=0.858 +[ALab02] 2023-06-11 19:35:43,263 (trainer:721) INFO: 64epoch:train:7164-7714batch: iter_time=2.287e-04, forward_time=0.066, loss_ctc=0.659, loss=0.659, backward_time=0.088, optim_step_time=0.039, optim0_lr0=2.233e-04, train_time=0.726 +[ALab02] 2023-06-11 19:37:42,245 (trainer:721) INFO: 64epoch:train:7715-8265batch: iter_time=2.895e-04, forward_time=0.080, loss_ctc=0.639, loss=0.639, backward_time=0.105, optim_step_time=0.048, optim0_lr0=2.232e-04, train_time=0.863 +[ALab02] 2023-06-11 19:39:31,193 (trainer:721) INFO: 64epoch:train:8266-8816batch: iter_time=2.575e-04, forward_time=0.072, loss_ctc=0.641, loss=0.641, backward_time=0.096, optim_step_time=0.044, optim0_lr0=2.231e-04, train_time=0.791 +[ALab02] 2023-06-11 19:41:10,909 (trainer:721) INFO: 64epoch:train:8817-9367batch: iter_time=2.319e-04, forward_time=0.065, loss_ctc=0.650, loss=0.650, backward_time=0.088, optim_step_time=0.040, optim0_lr0=2.230e-04, train_time=0.725 +[ALab02] 2023-06-11 19:42:49,536 (trainer:721) INFO: 64epoch:train:9368-9918batch: iter_time=1.988e-04, forward_time=0.065, loss_ctc=0.716, loss=0.716, backward_time=0.086, optim_step_time=0.041, optim0_lr0=2.229e-04, train_time=0.715 +[ALab02] 2023-06-11 19:44:27,136 (trainer:721) INFO: 64epoch:train:9919-10469batch: iter_time=2.242e-04, forward_time=0.063, loss_ctc=0.653, loss=0.653, backward_time=0.087, optim_step_time=0.039, optim0_lr0=2.228e-04, train_time=0.708 +[ALab02] 2023-06-11 19:46:48,050 (trainer:721) INFO: 64epoch:train:10470-11020batch: iter_time=4.612e-04, forward_time=0.094, loss_ctc=0.696, loss=0.696, backward_time=0.125, optim_step_time=0.057, optim0_lr0=2.227e-04, train_time=1.022 +[ALab02] 2023-06-11 19:47:47,577 (trainer:338) INFO: 64epoch results: [train] iter_time=3.926e-04, forward_time=0.082, loss_ctc=0.648, loss=0.648, backward_time=0.106, optim_step_time=0.049, optim0_lr0=2.236e-04, train_time=0.877, time=40 minutes and 19.39 seconds, total_count=705856, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.493, cer_ctc=0.054, cer=0.054, loss=4.493, time=26.2 seconds, total_count=28224, gpu_max_cached_mem_GB=10.861, [att_plot] time=30.35 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 19:47:50,011 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 19:47:50,012 (trainer:272) INFO: 65/70epoch started. Estimated time to finish: 5 hours, 10 minutes and 21.35 seconds +[ALab02] 2023-06-11 19:49:48,165 (trainer:721) INFO: 65epoch:train:1-551batch: iter_time=8.407e-04, forward_time=0.079, loss_ctc=0.685, loss=0.685, backward_time=0.103, optim_step_time=0.049, optim0_lr0=2.226e-04, train_time=0.855 +[ALab02] 2023-06-11 19:52:32,884 (trainer:721) INFO: 65epoch:train:552-1102batch: iter_time=4.654e-04, forward_time=0.113, loss_ctc=0.672, loss=0.672, backward_time=0.147, optim_step_time=0.068, optim0_lr0=2.226e-04, train_time=1.197 +[ALab02] 2023-06-11 19:55:03,524 (trainer:721) INFO: 65epoch:train:1103-1653batch: iter_time=3.718e-04, forward_time=0.106, loss_ctc=0.652, loss=0.652, backward_time=0.131, optim_step_time=0.062, optim0_lr0=2.225e-04, train_time=1.093 +[ALab02] 2023-06-11 19:57:31,743 (trainer:721) INFO: 65epoch:train:1654-2204batch: iter_time=3.589e-04, forward_time=0.102, loss_ctc=0.670, loss=0.670, backward_time=0.131, optim_step_time=0.060, optim0_lr0=2.224e-04, train_time=1.075 +[ALab02] 2023-06-11 19:59:43,416 (trainer:721) INFO: 65epoch:train:2205-2755batch: iter_time=3.358e-04, forward_time=0.091, loss_ctc=0.649, loss=0.649, backward_time=0.114, optim_step_time=0.055, optim0_lr0=2.223e-04, train_time=0.955 +[ALab02] 2023-06-11 20:02:12,143 (trainer:721) INFO: 65epoch:train:2756-3306batch: iter_time=3.631e-04, forward_time=0.104, loss_ctc=0.648, loss=0.648, backward_time=0.129, optim_step_time=0.062, optim0_lr0=2.222e-04, train_time=1.080 +[ALab02] 2023-06-11 20:04:36,478 (trainer:721) INFO: 65epoch:train:3307-3857batch: iter_time=3.590e-04, forward_time=0.098, loss_ctc=0.671, loss=0.671, backward_time=0.128, optim_step_time=0.059, optim0_lr0=2.221e-04, train_time=1.047 +[ALab02] 2023-06-11 20:06:42,551 (trainer:721) INFO: 65epoch:train:3858-4408batch: iter_time=3.334e-04, forward_time=0.086, loss_ctc=0.688, loss=0.688, backward_time=0.110, optim_step_time=0.053, optim0_lr0=2.220e-04, train_time=0.915 +[ALab02] 2023-06-11 20:09:08,569 (trainer:721) INFO: 65epoch:train:4409-4959batch: iter_time=3.651e-04, forward_time=0.100, loss_ctc=0.671, loss=0.671, backward_time=0.130, optim_step_time=0.060, optim0_lr0=2.220e-04, train_time=1.060 +[ALab02] 2023-06-11 20:11:29,096 (trainer:721) INFO: 65epoch:train:4960-5510batch: iter_time=3.601e-04, forward_time=0.098, loss_ctc=0.682, loss=0.682, backward_time=0.122, optim_step_time=0.060, optim0_lr0=2.219e-04, train_time=1.019 +[ALab02] 2023-06-11 20:13:38,581 (trainer:721) INFO: 65epoch:train:5511-6061batch: iter_time=3.178e-04, forward_time=0.088, loss_ctc=0.679, loss=0.679, backward_time=0.115, optim_step_time=0.052, optim0_lr0=2.218e-04, train_time=0.940 +[ALab02] 2023-06-11 20:16:03,334 (trainer:721) INFO: 65epoch:train:6062-6612batch: iter_time=3.402e-04, forward_time=0.098, loss_ctc=0.656, loss=0.656, backward_time=0.130, optim_step_time=0.059, optim0_lr0=2.217e-04, train_time=1.050 +[ALab02] 2023-06-11 20:18:25,577 (trainer:721) INFO: 65epoch:train:6613-7163batch: iter_time=3.542e-04, forward_time=0.098, loss_ctc=0.659, loss=0.659, backward_time=0.126, optim_step_time=0.057, optim0_lr0=2.216e-04, train_time=1.032 +[ALab02] 2023-06-11 20:21:24,199 (trainer:721) INFO: 65epoch:train:7164-7714batch: iter_time=4.209e-04, forward_time=0.122, loss_ctc=0.627, loss=0.627, backward_time=0.160, optim_step_time=0.073, optim0_lr0=2.215e-04, train_time=1.295 +[ALab02] 2023-06-11 20:23:55,739 (trainer:721) INFO: 65epoch:train:7715-8265batch: iter_time=3.883e-04, forward_time=0.105, loss_ctc=0.658, loss=0.658, backward_time=0.133, optim_step_time=0.062, optim0_lr0=2.214e-04, train_time=1.101 +[ALab02] 2023-06-11 20:26:06,025 (trainer:721) INFO: 65epoch:train:8266-8816batch: iter_time=3.363e-04, forward_time=0.088, loss_ctc=0.637, loss=0.637, backward_time=0.115, optim_step_time=0.055, optim0_lr0=2.214e-04, train_time=0.945 +[ALab02] 2023-06-11 20:28:14,986 (trainer:721) INFO: 65epoch:train:8817-9367batch: iter_time=3.252e-04, forward_time=0.089, loss_ctc=0.664, loss=0.664, backward_time=0.112, optim_step_time=0.054, optim0_lr0=2.213e-04, train_time=0.936 +[ALab02] 2023-06-11 20:30:23,844 (trainer:721) INFO: 65epoch:train:9368-9918batch: iter_time=3.049e-04, forward_time=0.088, loss_ctc=0.676, loss=0.676, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.212e-04, train_time=0.935 +[ALab02] 2023-06-11 20:32:44,556 (trainer:721) INFO: 65epoch:train:9919-10469batch: iter_time=3.326e-04, forward_time=0.095, loss_ctc=0.664, loss=0.664, backward_time=0.125, optim_step_time=0.058, optim0_lr0=2.211e-04, train_time=1.020 +[ALab02] 2023-06-11 20:35:00,833 (trainer:721) INFO: 65epoch:train:10470-11020batch: iter_time=3.317e-04, forward_time=0.094, loss_ctc=0.624, loss=0.624, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.210e-04, train_time=0.989 +[ALab02] 2023-06-11 20:36:23,141 (trainer:338) INFO: 65epoch results: [train] iter_time=3.801e-04, forward_time=0.097, loss_ctc=0.662, loss=0.662, backward_time=0.125, optim_step_time=0.058, optim0_lr0=2.218e-04, train_time=1.027, time=47 minutes and 13.69 seconds, total_count=716885, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.304, cer_ctc=0.052, cer=0.052, loss=4.304, time=31.3 seconds, total_count=28665, gpu_max_cached_mem_GB=10.861, [att_plot] time=48.13 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 20:36:26,511 (trainer:386) INFO: The best model has been updated: valid.cer +[ALab02] 2023-06-11 20:36:26,516 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/39epoch.pth, exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/64epoch.pth +[ALab02] 2023-06-11 20:36:26,516 (trainer:272) INFO: 66/70epoch started. Estimated time to finish: 4 hours, 18 minutes and 23.41 seconds +[ALab02] 2023-06-11 20:38:35,742 (trainer:721) INFO: 66epoch:train:1-551batch: iter_time=0.002, forward_time=0.086, loss_ctc=0.613, loss=0.613, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.209e-04, train_time=0.938 +[ALab02] 2023-06-11 20:40:48,884 (trainer:721) INFO: 66epoch:train:552-1102batch: iter_time=5.279e-04, forward_time=0.089, loss_ctc=0.648, loss=0.648, backward_time=0.119, optim_step_time=0.057, optim0_lr0=2.208e-04, train_time=0.966 +[ALab02] 2023-06-11 20:42:57,845 (trainer:721) INFO: 66epoch:train:1103-1653batch: iter_time=3.496e-04, forward_time=0.085, loss_ctc=0.596, loss=0.596, backward_time=0.116, optim_step_time=0.052, optim0_lr0=2.208e-04, train_time=0.935 +[ALab02] 2023-06-11 20:45:06,601 (trainer:721) INFO: 66epoch:train:1654-2204batch: iter_time=3.458e-04, forward_time=0.085, loss_ctc=0.630, loss=0.630, backward_time=0.116, optim_step_time=0.053, optim0_lr0=2.207e-04, train_time=0.934 +[ALab02] 2023-06-11 20:47:08,672 (trainer:721) INFO: 66epoch:train:2205-2755batch: iter_time=3.280e-04, forward_time=0.081, loss_ctc=0.644, loss=0.644, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.206e-04, train_time=0.883 +[ALab02] 2023-06-11 20:49:27,936 (trainer:721) INFO: 66epoch:train:2756-3306batch: iter_time=3.561e-04, forward_time=0.095, loss_ctc=0.625, loss=0.625, backward_time=0.124, optim_step_time=0.057, optim0_lr0=2.205e-04, train_time=1.013 +[ALab02] 2023-06-11 20:51:39,581 (trainer:721) INFO: 66epoch:train:3307-3857batch: iter_time=3.215e-04, forward_time=0.090, loss_ctc=0.617, loss=0.617, backward_time=0.115, optim_step_time=0.056, optim0_lr0=2.204e-04, train_time=0.955 +[ALab02] 2023-06-11 20:53:42,480 (trainer:721) INFO: 66epoch:train:3858-4408batch: iter_time=3.039e-04, forward_time=0.084, loss_ctc=0.623, loss=0.623, backward_time=0.107, optim_step_time=0.051, optim0_lr0=2.203e-04, train_time=0.892 +[ALab02] 2023-06-11 20:55:49,909 (trainer:721) INFO: 66epoch:train:4409-4959batch: iter_time=3.331e-04, forward_time=0.086, loss_ctc=0.642, loss=0.642, backward_time=0.113, optim_step_time=0.051, optim0_lr0=2.202e-04, train_time=0.926 +[ALab02] 2023-06-11 20:57:46,932 (trainer:721) INFO: 66epoch:train:4960-5510batch: iter_time=3.059e-04, forward_time=0.078, loss_ctc=0.649, loss=0.649, backward_time=0.103, optim_step_time=0.048, optim0_lr0=2.202e-04, train_time=0.849 +[ALab02] 2023-06-11 20:59:40,741 (trainer:721) INFO: 66epoch:train:5511-6061batch: iter_time=2.895e-04, forward_time=0.077, loss_ctc=0.673, loss=0.673, backward_time=0.100, optim_step_time=0.046, optim0_lr0=2.201e-04, train_time=0.826 +[ALab02] 2023-06-11 21:01:31,789 (trainer:721) INFO: 66epoch:train:6062-6612batch: iter_time=2.558e-04, forward_time=0.074, loss_ctc=0.642, loss=0.642, backward_time=0.098, optim_step_time=0.043, optim0_lr0=2.200e-04, train_time=0.805 +[ALab02] 2023-06-11 21:03:25,527 (trainer:721) INFO: 66epoch:train:6613-7163batch: iter_time=2.916e-04, forward_time=0.077, loss_ctc=0.631, loss=0.631, backward_time=0.099, optim_step_time=0.046, optim0_lr0=2.199e-04, train_time=0.825 +[ALab02] 2023-06-11 21:05:11,711 (trainer:721) INFO: 66epoch:train:7164-7714batch: iter_time=2.506e-04, forward_time=0.069, loss_ctc=0.634, loss=0.634, backward_time=0.095, optim_step_time=0.041, optim0_lr0=2.198e-04, train_time=0.771 +[ALab02] 2023-06-11 21:06:48,154 (trainer:721) INFO: 66epoch:train:7715-8265batch: iter_time=2.129e-04, forward_time=0.063, loss_ctc=0.614, loss=0.614, backward_time=0.085, optim_step_time=0.039, optim0_lr0=2.197e-04, train_time=0.700 +[ALab02] 2023-06-11 21:08:23,933 (trainer:721) INFO: 66epoch:train:8266-8816batch: iter_time=1.886e-04, forward_time=0.062, loss_ctc=0.605, loss=0.605, backward_time=0.085, optim_step_time=0.039, optim0_lr0=2.197e-04, train_time=0.695 +[ALab02] 2023-06-11 21:10:04,557 (trainer:721) INFO: 66epoch:train:8817-9367batch: iter_time=2.263e-04, forward_time=0.065, loss_ctc=0.615, loss=0.615, backward_time=0.090, optim_step_time=0.041, optim0_lr0=2.196e-04, train_time=0.731 +[ALab02] 2023-06-11 21:11:41,762 (trainer:721) INFO: 66epoch:train:9368-9918batch: iter_time=2.231e-04, forward_time=0.063, loss_ctc=0.617, loss=0.617, backward_time=0.087, optim_step_time=0.039, optim0_lr0=2.195e-04, train_time=0.705 +[ALab02] 2023-06-11 21:13:50,202 (trainer:721) INFO: 66epoch:train:9919-10469batch: iter_time=2.920e-04, forward_time=0.087, loss_ctc=0.608, loss=0.608, backward_time=0.114, optim_step_time=0.052, optim0_lr0=2.194e-04, train_time=0.930 +[ALab02] 2023-06-11 21:16:11,947 (trainer:721) INFO: 66epoch:train:10470-11020batch: iter_time=3.431e-04, forward_time=0.097, loss_ctc=0.659, loss=0.659, backward_time=0.126, optim_step_time=0.057, optim0_lr0=2.193e-04, train_time=1.029 +[ALab02] 2023-06-11 21:17:51,672 (trainer:338) INFO: 66epoch results: [train] iter_time=3.916e-04, forward_time=0.080, loss_ctc=0.629, loss=0.629, backward_time=0.106, optim_step_time=0.049, optim0_lr0=2.201e-04, train_time=0.865, time=39 minutes and 48.02 seconds, total_count=727914, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.278, cer_ctc=0.053, cer=0.053, loss=4.278, time=35.69 seconds, total_count=29106, gpu_max_cached_mem_GB=10.861, [att_plot] time=1 minute and 1.44 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 21:17:55,924 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 21:17:55,930 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/46epoch.pth +[ALab02] 2023-06-11 21:17:55,930 (trainer:272) INFO: 67/70epoch started. Estimated time to finish: 3 hours, 26 minutes and 5.68 seconds +[ALab02] 2023-06-11 21:20:21,381 (trainer:721) INFO: 67epoch:train:1-551batch: iter_time=0.002, forward_time=0.102, loss_ctc=0.616, loss=0.616, backward_time=0.124, optim_step_time=0.061, optim0_lr0=2.192e-04, train_time=1.056 +[ALab02] 2023-06-11 21:22:41,364 (trainer:721) INFO: 67epoch:train:552-1102batch: iter_time=3.565e-04, forward_time=0.098, loss_ctc=0.622, loss=0.622, backward_time=0.122, optim_step_time=0.058, optim0_lr0=2.192e-04, train_time=1.016 +[ALab02] 2023-06-11 21:24:59,913 (trainer:721) INFO: 67epoch:train:1103-1653batch: iter_time=3.566e-04, forward_time=0.094, loss_ctc=0.620, loss=0.620, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.191e-04, train_time=1.005 +[ALab02] 2023-06-11 21:27:21,414 (trainer:721) INFO: 67epoch:train:1654-2204batch: iter_time=3.393e-04, forward_time=0.099, loss_ctc=0.647, loss=0.647, backward_time=0.122, optim_step_time=0.060, optim0_lr0=2.190e-04, train_time=1.026 +[ALab02] 2023-06-11 21:29:42,601 (trainer:721) INFO: 67epoch:train:2205-2755batch: iter_time=3.414e-04, forward_time=0.099, loss_ctc=0.648, loss=0.648, backward_time=0.123, optim_step_time=0.058, optim0_lr0=2.189e-04, train_time=1.024 +[ALab02] 2023-06-11 21:31:50,133 (trainer:721) INFO: 67epoch:train:2756-3306batch: iter_time=3.287e-04, forward_time=0.088, loss_ctc=0.625, loss=0.625, backward_time=0.110, optim_step_time=0.054, optim0_lr0=2.188e-04, train_time=0.926 +[ALab02] 2023-06-11 21:34:18,031 (trainer:721) INFO: 67epoch:train:3307-3857batch: iter_time=3.560e-04, forward_time=0.102, loss_ctc=0.636, loss=0.636, backward_time=0.130, optim_step_time=0.061, optim0_lr0=2.187e-04, train_time=1.073 +[ALab02] 2023-06-11 21:36:36,422 (trainer:721) INFO: 67epoch:train:3858-4408batch: iter_time=3.432e-04, forward_time=0.095, loss_ctc=0.609, loss=0.609, backward_time=0.122, optim_step_time=0.057, optim0_lr0=2.187e-04, train_time=1.004 +[ALab02] 2023-06-11 21:38:42,732 (trainer:721) INFO: 67epoch:train:4409-4959batch: iter_time=3.104e-04, forward_time=0.086, loss_ctc=0.613, loss=0.613, backward_time=0.111, optim_step_time=0.052, optim0_lr0=2.186e-04, train_time=0.918 +[ALab02] 2023-06-11 21:41:06,229 (trainer:721) INFO: 67epoch:train:4960-5510batch: iter_time=3.551e-04, forward_time=0.098, loss_ctc=0.590, loss=0.590, backward_time=0.127, optim_step_time=0.059, optim0_lr0=2.185e-04, train_time=1.040 +[ALab02] 2023-06-11 21:43:20,188 (trainer:721) INFO: 67epoch:train:5511-6061batch: iter_time=3.246e-04, forward_time=0.092, loss_ctc=0.601, loss=0.601, backward_time=0.117, optim_step_time=0.057, optim0_lr0=2.184e-04, train_time=0.972 +[ALab02] 2023-06-11 21:45:30,502 (trainer:721) INFO: 67epoch:train:6062-6612batch: iter_time=3.100e-04, forward_time=0.089, loss_ctc=0.593, loss=0.593, backward_time=0.115, optim_step_time=0.053, optim0_lr0=2.183e-04, train_time=0.945 +[ALab02] 2023-06-11 21:47:39,095 (trainer:721) INFO: 67epoch:train:6613-7163batch: iter_time=3.123e-04, forward_time=0.088, loss_ctc=0.640, loss=0.640, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.183e-04, train_time=0.934 +[ALab02] 2023-06-11 21:50:00,433 (trainer:721) INFO: 67epoch:train:7164-7714batch: iter_time=3.227e-04, forward_time=0.096, loss_ctc=0.656, loss=0.656, backward_time=0.126, optim_step_time=0.059, optim0_lr0=2.182e-04, train_time=1.025 +[ALab02] 2023-06-11 21:52:16,983 (trainer:721) INFO: 67epoch:train:7715-8265batch: iter_time=3.282e-04, forward_time=0.093, loss_ctc=0.596, loss=0.596, backward_time=0.121, optim_step_time=0.057, optim0_lr0=2.181e-04, train_time=0.991 +[ALab02] 2023-06-11 21:54:32,020 (trainer:721) INFO: 67epoch:train:8266-8816batch: iter_time=3.252e-04, forward_time=0.093, loss_ctc=0.599, loss=0.599, backward_time=0.118, optim_step_time=0.056, optim0_lr0=2.180e-04, train_time=0.980 +[ALab02] 2023-06-11 21:56:33,207 (trainer:721) INFO: 67epoch:train:8817-9367batch: iter_time=2.879e-04, forward_time=0.082, loss_ctc=0.599, loss=0.599, backward_time=0.106, optim_step_time=0.049, optim0_lr0=2.179e-04, train_time=0.880 +[ALab02] 2023-06-11 21:58:55,379 (trainer:721) INFO: 67epoch:train:9368-9918batch: iter_time=3.711e-04, forward_time=0.098, loss_ctc=0.617, loss=0.617, backward_time=0.125, optim_step_time=0.059, optim0_lr0=2.178e-04, train_time=1.031 +[ALab02] 2023-06-11 22:01:08,569 (trainer:721) INFO: 67epoch:train:9919-10469batch: iter_time=3.051e-04, forward_time=0.090, loss_ctc=0.600, loss=0.600, backward_time=0.119, optim_step_time=0.055, optim0_lr0=2.178e-04, train_time=0.967 +[ALab02] 2023-06-11 22:03:09,308 (trainer:721) INFO: 67epoch:train:10470-11020batch: iter_time=2.837e-04, forward_time=0.082, loss_ctc=0.638, loss=0.638, backward_time=0.106, optim_step_time=0.049, optim0_lr0=2.177e-04, train_time=0.876 +[ALab02] 2023-06-11 22:04:35,052 (trainer:338) INFO: 67epoch results: [train] iter_time=4.001e-04, forward_time=0.093, loss_ctc=0.618, loss=0.618, backward_time=0.119, optim_step_time=0.056, optim0_lr0=2.185e-04, train_time=0.984, time=45 minutes and 15.66 seconds, total_count=738943, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.409, cer_ctc=0.053, cer=0.053, loss=4.409, time=29.4 seconds, total_count=29547, gpu_max_cached_mem_GB=10.861, [att_plot] time=54.05 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 22:04:39,151 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 22:04:39,154 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/57epoch.pth +[ALab02] 2023-06-11 22:04:39,155 (trainer:272) INFO: 68/70epoch started. Estimated time to finish: 2 hours, 34 minutes and 21.35 seconds +[ALab02] 2023-06-11 22:06:51,979 (trainer:721) INFO: 68epoch:train:1-551batch: iter_time=0.002, forward_time=0.089, loss_ctc=0.633, loss=0.633, backward_time=0.118, optim_step_time=0.053, optim0_lr0=2.176e-04, train_time=0.963 +[ALab02] 2023-06-11 22:09:05,799 (trainer:721) INFO: 68epoch:train:552-1102batch: iter_time=5.097e-04, forward_time=0.092, loss_ctc=0.595, loss=0.595, backward_time=0.117, optim_step_time=0.056, optim0_lr0=2.175e-04, train_time=0.972 +[ALab02] 2023-06-11 22:11:04,053 (trainer:721) INFO: 68epoch:train:1103-1653batch: iter_time=3.138e-04, forward_time=0.079, loss_ctc=0.610, loss=0.610, backward_time=0.105, optim_step_time=0.048, optim0_lr0=2.174e-04, train_time=0.858 +[ALab02] 2023-06-11 22:13:26,516 (trainer:721) INFO: 68epoch:train:1654-2204batch: iter_time=3.311e-04, forward_time=0.095, loss_ctc=0.610, loss=0.610, backward_time=0.128, optim_step_time=0.059, optim0_lr0=2.174e-04, train_time=1.033 +[ALab02] 2023-06-11 22:15:42,463 (trainer:721) INFO: 68epoch:train:2205-2755batch: iter_time=3.239e-04, forward_time=0.093, loss_ctc=0.636, loss=0.636, backward_time=0.120, optim_step_time=0.056, optim0_lr0=2.173e-04, train_time=0.988 +[ALab02] 2023-06-11 22:17:42,620 (trainer:721) INFO: 68epoch:train:2756-3306batch: iter_time=2.985e-04, forward_time=0.081, loss_ctc=0.635, loss=0.635, backward_time=0.105, optim_step_time=0.049, optim0_lr0=2.172e-04, train_time=0.872 +[ALab02] 2023-06-11 22:19:56,278 (trainer:721) INFO: 68epoch:train:3307-3857batch: iter_time=3.177e-04, forward_time=0.091, loss_ctc=0.618, loss=0.618, backward_time=0.118, optim_step_time=0.054, optim0_lr0=2.171e-04, train_time=0.969 +[ALab02] 2023-06-11 22:22:03,405 (trainer:721) INFO: 68epoch:train:3858-4408batch: iter_time=3.001e-04, forward_time=0.087, loss_ctc=0.642, loss=0.642, backward_time=0.112, optim_step_time=0.052, optim0_lr0=2.170e-04, train_time=0.922 +[ALab02] 2023-06-11 22:24:03,967 (trainer:721) INFO: 68epoch:train:4409-4959batch: iter_time=2.934e-04, forward_time=0.082, loss_ctc=0.636, loss=0.636, backward_time=0.106, optim_step_time=0.048, optim0_lr0=2.170e-04, train_time=0.876 +[ALab02] 2023-06-11 22:25:49,104 (trainer:721) INFO: 68epoch:train:4960-5510batch: iter_time=2.417e-04, forward_time=0.070, loss_ctc=0.618, loss=0.618, backward_time=0.093, optim_step_time=0.041, optim0_lr0=2.169e-04, train_time=0.763 +[ALab02] 2023-06-11 22:27:35,866 (trainer:721) INFO: 68epoch:train:5511-6061batch: iter_time=2.296e-04, forward_time=0.070, loss_ctc=0.625, loss=0.625, backward_time=0.095, optim_step_time=0.043, optim0_lr0=2.168e-04, train_time=0.773 +[ALab02] 2023-06-11 22:29:26,205 (trainer:721) INFO: 68epoch:train:6062-6612batch: iter_time=2.764e-04, forward_time=0.072, loss_ctc=0.645, loss=0.645, backward_time=0.099, optim_step_time=0.043, optim0_lr0=2.167e-04, train_time=0.802 +[ALab02] 2023-06-11 22:31:08,402 (trainer:721) INFO: 68epoch:train:6613-7163batch: iter_time=2.409e-04, forward_time=0.066, loss_ctc=0.620, loss=0.620, backward_time=0.092, optim_step_time=0.040, optim0_lr0=2.166e-04, train_time=0.742 +[ALab02] 2023-06-11 22:32:44,632 (trainer:721) INFO: 68epoch:train:7164-7714batch: iter_time=2.217e-04, forward_time=0.062, loss_ctc=0.609, loss=0.609, backward_time=0.085, optim_step_time=0.039, optim0_lr0=2.166e-04, train_time=0.698 +[ALab02] 2023-06-11 22:34:21,265 (trainer:721) INFO: 68epoch:train:7715-8265batch: iter_time=2.021e-04, forward_time=0.063, loss_ctc=0.586, loss=0.586, backward_time=0.085, optim_step_time=0.040, optim0_lr0=2.165e-04, train_time=0.700 +[ALab02] 2023-06-11 22:35:57,700 (trainer:721) INFO: 68epoch:train:8266-8816batch: iter_time=2.212e-04, forward_time=0.062, loss_ctc=0.594, loss=0.594, backward_time=0.086, optim_step_time=0.039, optim0_lr0=2.164e-04, train_time=0.700 +[ALab02] 2023-06-11 22:38:10,607 (trainer:721) INFO: 68epoch:train:8817-9367batch: iter_time=2.977e-04, forward_time=0.089, loss_ctc=0.618, loss=0.618, backward_time=0.119, optim_step_time=0.054, optim0_lr0=2.163e-04, train_time=0.963 +[ALab02] 2023-06-11 22:40:29,731 (trainer:721) INFO: 68epoch:train:9368-9918batch: iter_time=3.428e-04, forward_time=0.096, loss_ctc=0.593, loss=0.593, backward_time=0.121, optim_step_time=0.057, optim0_lr0=2.162e-04, train_time=1.011 +[ALab02] 2023-06-11 22:43:06,658 (trainer:721) INFO: 68epoch:train:9919-10469batch: iter_time=3.730e-04, forward_time=0.109, loss_ctc=0.613, loss=0.613, backward_time=0.138, optim_step_time=0.065, optim0_lr0=2.162e-04, train_time=1.138 +[ALab02] 2023-06-11 22:45:33,222 (trainer:721) INFO: 68epoch:train:10470-11020batch: iter_time=3.396e-04, forward_time=0.101, loss_ctc=0.610, loss=0.610, backward_time=0.129, optim_step_time=0.059, optim0_lr0=2.161e-04, train_time=1.063 +[ALab02] 2023-06-11 22:46:59,418 (trainer:338) INFO: 68epoch results: [train] iter_time=3.818e-04, forward_time=0.082, loss_ctc=0.617, loss=0.617, backward_time=0.109, optim_step_time=0.050, optim0_lr0=2.168e-04, train_time=0.891, time=40 minutes and 57.09 seconds, total_count=749972, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.301, cer_ctc=0.052, cer=0.052, loss=4.301, time=35.6 seconds, total_count=29988, gpu_max_cached_mem_GB=10.861, [att_plot] time=47.58 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 22:47:02,952 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 22:47:02,954 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/58epoch.pth +[ALab02] 2023-06-11 22:47:02,954 (trainer:272) INFO: 69/70epoch started. Estimated time to finish: 1 hour, 42 minutes and 38.26 seconds +[ALab02] 2023-06-11 22:49:24,051 (trainer:721) INFO: 69epoch:train:1-551batch: iter_time=0.002, forward_time=0.096, loss_ctc=0.596, loss=0.596, backward_time=0.123, optim_step_time=0.059, optim0_lr0=2.160e-04, train_time=1.024 +[ALab02] 2023-06-11 22:51:48,692 (trainer:721) INFO: 69epoch:train:552-1102batch: iter_time=3.477e-04, forward_time=0.099, loss_ctc=0.580, loss=0.580, backward_time=0.128, optim_step_time=0.060, optim0_lr0=2.159e-04, train_time=1.050 +[ALab02] 2023-06-11 22:54:06,504 (trainer:721) INFO: 69epoch:train:1103-1653batch: iter_time=3.269e-04, forward_time=0.096, loss_ctc=0.588, loss=0.588, backward_time=0.121, optim_step_time=0.056, optim0_lr0=2.158e-04, train_time=1.000 +[ALab02] 2023-06-11 22:56:09,730 (trainer:721) INFO: 69epoch:train:1654-2204batch: iter_time=3.115e-04, forward_time=0.083, loss_ctc=0.611, loss=0.611, backward_time=0.108, optim_step_time=0.050, optim0_lr0=2.158e-04, train_time=0.894 +[ALab02] 2023-06-11 22:58:36,774 (trainer:721) INFO: 69epoch:train:2205-2755batch: iter_time=3.826e-04, forward_time=0.102, loss_ctc=0.606, loss=0.606, backward_time=0.129, optim_step_time=0.061, optim0_lr0=2.157e-04, train_time=1.068 +[ALab02] 2023-06-11 23:00:54,123 (trainer:721) INFO: 69epoch:train:2756-3306batch: iter_time=3.269e-04, forward_time=0.094, loss_ctc=0.605, loss=0.605, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.156e-04, train_time=0.996 +[ALab02] 2023-06-11 23:02:59,017 (trainer:721) INFO: 69epoch:train:3307-3857batch: iter_time=3.043e-04, forward_time=0.086, loss_ctc=0.591, loss=0.591, backward_time=0.109, optim_step_time=0.052, optim0_lr0=2.155e-04, train_time=0.906 +[ALab02] 2023-06-11 23:05:08,243 (trainer:721) INFO: 69epoch:train:3858-4408batch: iter_time=3.050e-04, forward_time=0.089, loss_ctc=0.645, loss=0.645, backward_time=0.113, optim_step_time=0.055, optim0_lr0=2.154e-04, train_time=0.938 +[ALab02] 2023-06-11 23:07:31,382 (trainer:721) INFO: 69epoch:train:4409-4959batch: iter_time=3.640e-04, forward_time=0.099, loss_ctc=0.660, loss=0.660, backward_time=0.127, optim_step_time=0.058, optim0_lr0=2.154e-04, train_time=1.040 +[ALab02] 2023-06-11 23:09:39,266 (trainer:721) INFO: 69epoch:train:4960-5510batch: iter_time=3.136e-04, forward_time=0.086, loss_ctc=0.638, loss=0.638, backward_time=0.114, optim_step_time=0.052, optim0_lr0=2.153e-04, train_time=0.928 +[ALab02] 2023-06-11 23:11:48,717 (trainer:721) INFO: 69epoch:train:5511-6061batch: iter_time=3.070e-04, forward_time=0.089, loss_ctc=0.614, loss=0.614, backward_time=0.113, optim_step_time=0.053, optim0_lr0=2.152e-04, train_time=0.939 +[ALab02] 2023-06-11 23:15:07,714 (trainer:721) INFO: 69epoch:train:6062-6612batch: iter_time=5.391e-04, forward_time=0.140, loss_ctc=0.607, loss=0.607, backward_time=0.174, optim_step_time=0.083, optim0_lr0=2.151e-04, train_time=1.443 +[ALab02] 2023-06-11 23:17:17,937 (trainer:721) INFO: 69epoch:train:6613-7163batch: iter_time=3.108e-04, forward_time=0.090, loss_ctc=0.606, loss=0.606, backward_time=0.114, optim_step_time=0.054, optim0_lr0=2.151e-04, train_time=0.946 +[ALab02] 2023-06-11 23:19:27,296 (trainer:721) INFO: 69epoch:train:7164-7714batch: iter_time=3.063e-04, forward_time=0.088, loss_ctc=0.597, loss=0.597, backward_time=0.113, optim_step_time=0.054, optim0_lr0=2.150e-04, train_time=0.938 +[ALab02] 2023-06-11 23:22:42,752 (trainer:721) INFO: 69epoch:train:7715-8265batch: iter_time=5.420e-04, forward_time=0.140, loss_ctc=0.577, loss=0.577, backward_time=0.169, optim_step_time=0.079, optim0_lr0=2.149e-04, train_time=1.417 +[ALab02] 2023-06-11 23:25:18,308 (trainer:721) INFO: 69epoch:train:8266-8816batch: iter_time=3.258e-04, forward_time=0.118, loss_ctc=0.604, loss=0.604, backward_time=0.123, optim_step_time=0.056, optim0_lr0=2.148e-04, train_time=1.129 +[ALab02] 2023-06-11 23:28:50,725 (trainer:721) INFO: 69epoch:train:8817-9367batch: iter_time=5.253e-04, forward_time=0.157, loss_ctc=0.604, loss=0.604, backward_time=0.178, optim_step_time=0.081, optim0_lr0=2.147e-04, train_time=1.545 +[ALab02] 2023-06-11 23:31:51,742 (trainer:721) INFO: 69epoch:train:9368-9918batch: iter_time=3.069e-04, forward_time=0.152, loss_ctc=0.603, loss=0.603, backward_time=0.125, optim_step_time=0.054, optim0_lr0=2.147e-04, train_time=1.311 +[ALab02] 2023-06-11 23:34:13,090 (trainer:721) INFO: 69epoch:train:9919-10469batch: iter_time=3.013e-04, forward_time=0.104, loss_ctc=0.590, loss=0.590, backward_time=0.116, optim_step_time=0.051, optim0_lr0=2.146e-04, train_time=1.028 +[ALab02] 2023-06-11 23:36:17,256 (trainer:721) INFO: 69epoch:train:10470-11020batch: iter_time=2.879e-04, forward_time=0.085, loss_ctc=0.580, loss=0.580, backward_time=0.108, optim_step_time=0.051, optim0_lr0=2.145e-04, train_time=0.901 +[ALab02] 2023-06-11 23:37:24,853 (trainer:338) INFO: 69epoch results: [train] iter_time=4.241e-04, forward_time=0.105, loss_ctc=0.605, loss=0.605, backward_time=0.126, optim_step_time=0.059, optim0_lr0=2.152e-04, train_time=1.072, time=49 minutes and 16.48 seconds, total_count=761001, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.337, cer_ctc=0.052, cer=0.052, loss=4.337, time=27.7 seconds, total_count=30429, gpu_max_cached_mem_GB=10.861, [att_plot] time=37.71 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-11 23:37:27,882 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-11 23:37:27,885 (trainer:440) INFO: The model files were removed: exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/52epoch.pth +[ALab02] 2023-06-11 23:37:27,886 (trainer:272) INFO: 70/70epoch started. Estimated time to finish: 51 minutes and 18.34 seconds +[ALab02] 2023-06-11 23:38:18,113 (trainer:663) WARNING: The grad norm is nan. Skipping updating the model. +[ALab02] 2023-06-11 23:39:44,023 (trainer:721) INFO: 70epoch:train:1-551batch: iter_time=0.002, forward_time=0.091, loss_ctc=0.576, loss=0.576, backward_time=0.120, optim_step_time=0.057, optim0_lr0=2.144e-04, train_time=0.989 +[ALab02] 2023-06-11 23:41:50,623 (trainer:721) INFO: 70epoch:train:552-1102batch: iter_time=2.990e-04, forward_time=0.087, loss_ctc=0.606, loss=0.606, backward_time=0.111, optim_step_time=0.052, optim0_lr0=2.144e-04, train_time=0.918 +[ALab02] 2023-06-11 23:41:53,007 (trainer:663) WARNING: The grad norm is nan. Skipping updating the model. +[ALab02] 2023-06-11 23:43:46,967 (trainer:721) INFO: 70epoch:train:1103-1653batch: iter_time=2.714e-04, forward_time=0.079, loss_ctc=0.597, loss=0.597, backward_time=0.102, optim_step_time=0.046, optim0_lr0=2.143e-04, train_time=0.844 +[ALab02] 2023-06-11 23:45:38,821 (trainer:721) INFO: 70epoch:train:1654-2204batch: iter_time=2.545e-04, forward_time=0.075, loss_ctc=0.597, loss=0.597, backward_time=0.099, optim_step_time=0.044, optim0_lr0=2.142e-04, train_time=0.811 +[ALab02] 2023-06-11 23:47:40,126 (trainer:721) INFO: 70epoch:train:2205-2755batch: iter_time=2.972e-04, forward_time=0.081, loss_ctc=0.563, loss=0.563, backward_time=0.107, optim_step_time=0.051, optim0_lr0=2.141e-04, train_time=0.881 +[ALab02] 2023-06-11 23:49:37,612 (trainer:721) INFO: 70epoch:train:2756-3306batch: iter_time=2.626e-04, forward_time=0.077, loss_ctc=0.562, loss=0.562, backward_time=0.106, optim_step_time=0.045, optim0_lr0=2.140e-04, train_time=0.852 +[ALab02] 2023-06-11 23:51:27,915 (trainer:721) INFO: 70epoch:train:3307-3857batch: iter_time=2.559e-04, forward_time=0.073, loss_ctc=0.592, loss=0.592, backward_time=0.098, optim_step_time=0.044, optim0_lr0=2.140e-04, train_time=0.801 +[ALab02] 2023-06-11 23:53:02,776 (trainer:721) INFO: 70epoch:train:3858-4408batch: iter_time=2.018e-04, forward_time=0.062, loss_ctc=0.602, loss=0.602, backward_time=0.084, optim_step_time=0.038, optim0_lr0=2.139e-04, train_time=0.688 +[ALab02] 2023-06-11 23:54:51,073 (trainer:721) INFO: 70epoch:train:4409-4959batch: iter_time=2.566e-04, forward_time=0.071, loss_ctc=0.624, loss=0.624, backward_time=0.096, optim_step_time=0.045, optim0_lr0=2.138e-04, train_time=0.786 +[ALab02] 2023-06-11 23:56:33,468 (trainer:721) INFO: 70epoch:train:4960-5510batch: iter_time=2.347e-04, forward_time=0.066, loss_ctc=0.638, loss=0.638, backward_time=0.091, optim_step_time=0.041, optim0_lr0=2.137e-04, train_time=0.743 +[ALab02] 2023-06-11 23:58:12,514 (trainer:721) INFO: 70epoch:train:5511-6061batch: iter_time=2.120e-04, forward_time=0.063, loss_ctc=0.625, loss=0.625, backward_time=0.088, optim_step_time=0.040, optim0_lr0=2.137e-04, train_time=0.719 +[ALab02] 2023-06-11 23:59:38,097 (trainer:721) INFO: 70epoch:train:6062-6612batch: iter_time=1.741e-04, forward_time=0.055, loss_ctc=0.609, loss=0.609, backward_time=0.075, optim_step_time=0.035, optim0_lr0=2.136e-04, train_time=0.621 +[ALab02] 2023-06-12 00:01:23,388 (trainer:721) INFO: 70epoch:train:6613-7163batch: iter_time=2.266e-04, forward_time=0.067, loss_ctc=0.624, loss=0.624, backward_time=0.096, optim_step_time=0.042, optim0_lr0=2.135e-04, train_time=0.764 +[ALab02] 2023-06-12 00:03:51,418 (trainer:721) INFO: 70epoch:train:7164-7714batch: iter_time=3.402e-04, forward_time=0.102, loss_ctc=0.649, loss=0.649, backward_time=0.131, optim_step_time=0.060, optim0_lr0=2.134e-04, train_time=1.072 +[ALab02] 2023-06-12 00:06:11,619 (trainer:721) INFO: 70epoch:train:7715-8265batch: iter_time=3.230e-04, forward_time=0.095, loss_ctc=0.648, loss=0.648, backward_time=0.125, optim_step_time=0.057, optim0_lr0=2.134e-04, train_time=1.018 +[ALab02] 2023-06-12 00:08:41,505 (trainer:721) INFO: 70epoch:train:8266-8816batch: iter_time=4.319e-04, forward_time=0.104, loss_ctc=0.649, loss=0.649, backward_time=0.131, optim_step_time=0.062, optim0_lr0=2.133e-04, train_time=1.087 +[ALab02] 2023-06-12 00:11:00,464 (trainer:721) INFO: 70epoch:train:8817-9367batch: iter_time=3.273e-04, forward_time=0.095, loss_ctc=0.647, loss=0.647, backward_time=0.123, optim_step_time=0.056, optim0_lr0=2.132e-04, train_time=1.009 +[ALab02] 2023-06-12 00:13:09,655 (trainer:721) INFO: 70epoch:train:9368-9918batch: iter_time=3.097e-04, forward_time=0.089, loss_ctc=0.609, loss=0.609, backward_time=0.112, optim_step_time=0.053, optim0_lr0=2.131e-04, train_time=0.937 +[ALab02] 2023-06-12 00:15:24,600 (trainer:721) INFO: 70epoch:train:9919-10469batch: iter_time=3.281e-04, forward_time=0.092, loss_ctc=0.619, loss=0.619, backward_time=0.118, optim_step_time=0.057, optim0_lr0=2.130e-04, train_time=0.979 +[ALab02] 2023-06-12 00:17:37,619 (trainer:721) INFO: 70epoch:train:10470-11020batch: iter_time=3.157e-04, forward_time=0.092, loss_ctc=0.598, loss=0.598, backward_time=0.116, optim_step_time=0.055, optim0_lr0=2.130e-04, train_time=0.965 +[ALab02] 2023-06-12 00:19:02,150 (trainer:338) INFO: 70epoch results: [train] iter_time=3.429e-04, forward_time=0.081, loss_ctc=0.612, loss=0.612, backward_time=0.107, optim_step_time=0.049, optim0_lr0=2.137e-04, train_time=0.874, time=40 minutes and 12.49 seconds, total_count=772030, gpu_max_cached_mem_GB=10.861, [valid] loss_ctc=4.446, cer_ctc=0.053, cer=0.053, loss=4.446, time=33.88 seconds, total_count=30870, gpu_max_cached_mem_GB=10.861, [att_plot] time=47.9 seconds, total_count=0, gpu_max_cached_mem_GB=10.861 +[ALab02] 2023-06-12 00:19:06,556 (trainer:384) INFO: There are no improvements in this epoch +[ALab02] 2023-06-12 00:19:06,557 (trainer:458) INFO: The training was finished at 70 epochs +[ALab02] 2023-06-12 00:19:06,615 (average_nbest_models:69) INFO: Averaging 10best models: criterion="valid.cer": exp_uma_conformer_12e_69/asr_train_asr_unimodal_conformer_raw_zh_char_sp/valid.cer.ave_10best.pth +# Accounting: time=214922 threads=1 +# Ended (code 0) at Mon Jun 12 00:19:15 CST 2023, elapsed time 214922 seconds