Upload 4 files

stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:838561f19ffb3237af9ccf5fb7c15483e79b401c0e5187a5a010218c71186cda
+size 4045597881

stability/2024-07-05__17-35-31/config.yaml

@@ -0,0 +1,119 @@
+fix_seed: 0
+checkpoints_every: 64
+tensorboard_log: True
+tqdm_progress_bar: False
+result_path: ./results/stability/
+
+resume:
+  resume: False
+  resume_path: /path/to/checkpoint.pth
+  restart_optimizer: True
+  handle_shape_missmatch: True
+
+prot2token_model:
+  compile_model: True
+  positional_encoding_type: learned # absolute, learned
+  protein_encoder:
+    model_name:  facebook/esm2_t33_650M_UR50D # facebook/esm2_t33_650M_UR50D, facebook/esm2_t30_150M_UR50D, facebook/esm2_t12_35M_UR50D, facebook/esm2_t6_8M_UR50D, Rostlab/prot_t5_base_mt_uniref50
+    max_len: 1024
+    drop_positional_encoding: True
+    quantization_4_bit: False # use with tune_embedding enabled
+    tune_embedding: False
+    fine_tune:
+      enable: True
+      last_layers_trainable: 6
+    lora:
+      enable: False
+      r: 8
+      lora_alpha: 32
+      lora_dropout: 0.05
+  molecule_encoder:
+    enable: False
+    model_name: gayane/BARTSmiles
+    max_len: 128
+    drop_positional_encoding: True
+    tune_embedding: False
+    fine_tune:
+      enable: False
+      last_layers_trainable: 2
+  decoder:
+    dimension: 640
+    dim_feedforward: 1280
+    num_heads: 8
+    num_layers: 8
+    max_len: 32
+    activation_function: gelu
+
+train_settings:
+  skip: False
+  data_path: /mnt/hdd8/mehdi/datasets/Joint_training/
+  num_epochs: 48
+  start_metric_epoch: 2
+  shuffle: True
+  loss: crossentropy # crossentropy or focal
+  sample_weight: True
+  task_weight: False
+  mixed_precision: bf16 # no, fp16, bf16, fp8
+  device: cuda
+  batch_size: 16
+  random_masking: 0.15 # 0 to 1.0
+  num_workers: 0
+  grad_accumulation: 32
+  max_task_samples: 250000
+
+valid_settings:
+  data_path: /mnt/hdd8/mehdi/datasets/Joint_training/
+  do_every: 1
+  batch_size: 1
+  num_workers: 0
+
+test_settings:
+  enable: True
+  data_path: /mnt/hdd8/mehdi/datasets/Joint_training/
+  batch_size: 1
+  num_workers: 0
+  monitoring_metrics:
+    fluorescence: spearman
+    stability: spearman
+    protein_ligand_affinity: rmse
+    human_ppi: f1
+    structure_similarity: spearman
+    protein_protein_interface: auc
+    enzyme_reaction: f1
+    fold: f1
+    localization_deeploc: macro_f1
+    secondary_structure: accuracy
+
+optimizer:
+  name: adam
+  lr: 5e-5
+  weight_decouple: True
+  weight_decay: 1e-2
+  eps: 1e-16
+  beta_1: 0.9
+  beta_2: 0.999
+  use_8bit_adam: False
+  grad_clip_norm: 1
+  decay:
+    warmup: 256
+    min_lr: 1e-6
+    gamma: 0.2
+    num_restarts: 1
+
+tasks:
+  phosphorylation: False
+  localization: False
+  localization_deeploc: False
+  fold: False
+  enzyme_reaction: False
+  human_ppi: False
+  structure_similarity: False
+  protein_protein_interface: False # decoder max length: 210
+  protein_ligand_affinity: False
+  fluorescence: False
+  stability: True
+  amino_to_fold_seek: False # decoder max length: max input length + 1
+  secondary_structure: False # decoder max length: max input length + 1
+  gene_ontology: False # decoder max length: 625
+  enzyme_commission: False # decoder max length: 47
+  auxiliary: False

stability/2024-07-05__17-35-31/decoder_tokenizer.yaml

@@ -0,0 +1,15 @@
+.: 14
+'0': 4
+'1': 5
+'2': 6
+'3': 7
+'4': 8
+'5': 9
+'6': 10
+'7': 11
+'8': 12
+'9': 13
+<bos>: 1
+<eos>: 2
+<pad>: 0
+<task_stability>: 3

stability/2024-07-05__17-35-31/logs.txt

@@ -0,0 +1,224 @@
+2024-07-05 17:35:31,450 - <task_stability>: remaining samples: 53614
+2024-07-05 17:35:31,744 - <task_stability>: remaining samples: 2512
+2024-07-05 17:35:31,781 - <task_stability>: remaining samples: 12851
+2024-07-05 17:35:31,782 - preparing dataloaders are done
+2024-07-05 17:35:33,302 - protein encoder trainable params:  118,887,040 || all params:  653,173,781 || trainable%: 18.20144094240672
+2024-07-05 17:35:33,396 - decoder trainable params:  40,103,055 || all params:  40,103,055 || trainable%: 100.0
+2024-07-05 17:35:33,397 - supermodel trainable params:  158,990,095 || all params:  693,276,836 || trainable%: 22.93313244350198
+2024-07-05 17:35:33,397 - preparing model is done
+2024-07-05 17:35:33,399 - preparing optimizer is done
+2024-07-05 17:35:37,350 - compile model is done
+2024-07-05 17:35:37,352 - number of train steps per epoch: 105
+2024-07-05 17:35:37,352 - number of stability valid steps per epoch: 2512
+2024-07-05 18:05:09,177 - epoch 1/48 - steps 105, - time 1771.82s, train loss 0.381185
+2024-07-05 18:31:11,659 - epoch 2/48 - steps 105, - time 1562.48s, train loss 0.26480179
+2024-07-05 18:57:15,738 - epoch 3/48 - steps 105, - time 1563.72s, train loss 0.23365109
+2024-07-05 18:57:15,752 - 	train stability spearman p 0.0302, mae 0.0882, number of correct prediction 53149
+2024-07-05 19:01:15,160 - evaluation dataset 1 - steps 2512 - time 239.36s, valid loss 0.33956882
+2024-07-05 19:01:15,160 - 	valid stability spearman p 0.6081, mae 0.0689, number of correct prediction 2512
+2024-07-05 19:01:20,149 - 	new best spearman for stability:  0.6081
+2024-07-05 19:01:20,149 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 19:28:23,082 - epoch 4/48 - steps 105, - time 1622.54s, train loss 0.20023499
+2024-07-05 19:28:23,082 - 	train stability spearman p 0.0698, mae 0.086, number of correct prediction 53182
+2024-07-05 19:31:30,154 - evaluation dataset 1 - steps 2512 - time 187.04s, valid loss 0.40229009
+2024-07-05 19:31:30,154 - 	valid stability spearman p 0.6449, mae 0.0649, number of correct prediction 2512
+2024-07-05 19:33:34,804 - 	new best spearman for stability:  0.6449
+2024-07-05 19:33:34,805 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 20:01:54,924 - epoch 5/48 - steps 105, - time 1699.73s, train loss 0.18428825
+2024-07-05 20:01:54,935 - 	train stability spearman p 0.1295, mae 0.0843, number of correct prediction 53204
+2024-07-05 20:05:01,870 - evaluation dataset 1 - steps 2512 - time 186.91s, valid loss 0.44731443
+2024-07-05 20:05:01,871 - 	valid stability spearman p 0.7055, mae 0.0594, number of correct prediction 2512
+2024-07-05 20:05:22,357 - 	new best spearman for stability:  0.7055
+2024-07-05 20:05:22,358 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 20:32:13,923 - epoch 6/48 - steps 105, - time 1611.15s, train loss 0.17623341
+2024-07-05 20:32:13,923 - 	train stability spearman p 0.2035, mae 0.0829, number of correct prediction 53231
+2024-07-05 20:35:12,223 - evaluation dataset 1 - steps 2512 - time 178.27s, valid loss 0.46854883
+2024-07-05 20:35:12,224 - 	valid stability spearman p 0.7019, mae 0.0642, number of correct prediction 2512
+2024-07-05 21:00:46,165 - epoch 7/48 - steps 105, - time 1533.53s, train loss 0.17145256
+2024-07-05 21:00:46,165 - 	train stability spearman p 0.2589, mae 0.0811, number of correct prediction 53241
+2024-07-05 21:03:44,364 - evaluation dataset 1 - steps 2512 - time 178.17s, valid loss 0.50273507
+2024-07-05 21:03:44,364 - 	valid stability spearman p 0.712, mae 0.0584, number of correct prediction 2512
+2024-07-05 21:04:04,732 - 	new best spearman for stability:  0.712
+2024-07-05 21:04:04,732 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 21:29:37,243 - epoch 8/48 - steps 105, - time 1532.1s, train loss 0.16834861
+2024-07-05 21:29:37,244 - 	train stability spearman p 0.2966, mae 0.0798, number of correct prediction 53239
+2024-07-05 21:32:34,979 - evaluation dataset 1 - steps 2512 - time 177.7s, valid loss 0.49872354
+2024-07-05 21:32:34,979 - 	valid stability spearman p 0.7226, mae 0.0581, number of correct prediction 2512
+2024-07-05 21:32:55,539 - 	new best spearman for stability:  0.7226
+2024-07-05 21:32:55,539 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 21:58:29,621 - epoch 9/48 - steps 105, - time 1533.67s, train loss 0.16531481
+2024-07-05 21:58:29,621 - 	train stability spearman p 0.3378, mae 0.0776, number of correct prediction 53276
+2024-07-05 22:01:26,298 - evaluation dataset 1 - steps 2512 - time 176.65s, valid loss 0.51312769
+2024-07-05 22:01:26,298 - 	valid stability spearman p 0.7233, mae 0.0663, number of correct prediction 2512
+2024-07-05 22:01:46,973 - 	new best spearman for stability:  0.7233
+2024-07-05 22:01:46,973 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 22:27:21,344 - epoch 10/48 - steps 105, - time 1533.94s, train loss 0.16316066
+2024-07-05 22:27:21,344 - 	train stability spearman p 0.3724, mae 0.0764, number of correct prediction 53262
+2024-07-05 22:30:18,295 - evaluation dataset 1 - steps 2512 - time 176.92s, valid loss 0.54175883
+2024-07-05 22:30:18,295 - 	valid stability spearman p 0.7437, mae 0.054, number of correct prediction 2512
+2024-07-05 22:30:38,703 - 	new best spearman for stability:  0.7437
+2024-07-05 22:30:38,704 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-05 22:56:12,609 - epoch 11/48 - steps 105, - time 1533.48s, train loss 0.16117712
+2024-07-05 22:56:12,609 - 	train stability spearman p 0.4078, mae 0.0742, number of correct prediction 53271
+2024-07-05 22:59:09,976 - evaluation dataset 1 - steps 2512 - time 177.34s, valid loss 0.55451429
+2024-07-05 22:59:09,977 - 	valid stability spearman p 0.7354, mae 0.0547, number of correct prediction 2512
+2024-07-05 23:24:41,255 - epoch 12/48 - steps 105, - time 1530.85s, train loss 0.15981591
+2024-07-05 23:24:41,255 - 	train stability spearman p 0.4316, mae 0.0727, number of correct prediction 53269
+2024-07-05 23:27:38,432 - evaluation dataset 1 - steps 2512 - time 177.14s, valid loss 0.53740785
+2024-07-05 23:27:38,432 - 	valid stability spearman p 0.7315, mae 0.0561, number of correct prediction 2512
+2024-07-05 23:54:00,607 - epoch 13/48 - steps 105, - time 1581.75s, train loss 0.15810851
+2024-07-05 23:54:00,608 - 	train stability spearman p 0.4586, mae 0.0709, number of correct prediction 53294
+2024-07-05 23:56:56,755 - evaluation dataset 1 - steps 2512 - time 176.11s, valid loss 0.57103281
+2024-07-05 23:56:56,755 - 	valid stability spearman p 0.7281, mae 0.0544, number of correct prediction 2512
+2024-07-06 00:22:39,173 - epoch 14/48 - steps 105, - time 1541.97s, train loss 0.15659422
+2024-07-06 00:22:39,174 - 	train stability spearman p 0.4792, mae 0.0695, number of correct prediction 53290
+2024-07-06 00:25:30,633 - evaluation dataset 1 - steps 2512 - time 171.43s, valid loss 0.58697268
+2024-07-06 00:25:30,633 - 	valid stability spearman p 0.7403, mae 0.0551, number of correct prediction 2512
+2024-07-06 00:51:12,854 - epoch 15/48 - steps 105, - time 1541.69s, train loss 0.15534321
+2024-07-06 00:51:12,854 - 	train stability spearman p 0.5026, mae 0.0675, number of correct prediction 53310
+2024-07-06 00:54:10,545 - evaluation dataset 1 - steps 2512 - time 177.66s, valid loss 0.56736496
+2024-07-06 00:54:10,545 - 	valid stability spearman p 0.7459, mae 0.0528, number of correct prediction 2512
+2024-07-06 00:54:29,250 - 	new best spearman for stability:  0.7459
+2024-07-06 00:54:29,252 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-06 01:20:12,180 - epoch 16/48 - steps 105, - time 1542.49s, train loss 0.15383106
+2024-07-06 01:20:12,181 - 	train stability spearman p 0.5228, mae 0.0662, number of correct prediction 53276
+2024-07-06 01:23:25,122 - evaluation dataset 1 - steps 2512 - time 192.78s, valid loss 0.591676
+2024-07-06 01:23:25,122 - 	valid stability spearman p 0.7357, mae 0.053, number of correct prediction 2512
+2024-07-06 01:49:12,194 - epoch 17/48 - steps 105, - time 1546.63s, train loss 0.15279829
+2024-07-06 01:49:12,194 - 	train stability spearman p 0.5419, mae 0.0646, number of correct prediction 53304
+2024-07-06 01:52:08,951 - evaluation dataset 1 - steps 2512 - time 176.72s, valid loss 0.59653156
+2024-07-06 01:52:08,952 - 	valid stability spearman p 0.7379, mae 0.0531, number of correct prediction 2512
+2024-07-06 02:17:50,682 - epoch 18/48 - steps 105, - time 1541.29s, train loss 0.15158923
+2024-07-06 02:17:50,682 - 	train stability spearman p 0.5574, mae 0.0631, number of correct prediction 53302
+2024-07-06 02:20:48,944 - evaluation dataset 1 - steps 2512 - time 178.23s, valid loss 0.592355
+2024-07-06 02:20:48,944 - 	valid stability spearman p 0.7279, mae 0.0552, number of correct prediction 2512
+2024-07-06 02:46:33,038 - epoch 19/48 - steps 105, - time 1543.65s, train loss 0.15060912
+2024-07-06 02:46:33,038 - 	train stability spearman p 0.5698, mae 0.0619, number of correct prediction 53298
+2024-07-06 02:49:30,836 - evaluation dataset 1 - steps 2512 - time 177.76s, valid loss 0.59448483
+2024-07-06 02:49:30,837 - 	valid stability spearman p 0.7466, mae 0.0526, number of correct prediction 2512
+2024-07-06 02:49:49,121 - 	new best spearman for stability:  0.7466
+2024-07-06 02:49:49,122 - 	saving the best model in /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-06 03:15:31,367 - epoch 20/48 - steps 105, - time 1541.8s, train loss 0.14970614
+2024-07-06 03:15:31,367 - 	train stability spearman p 0.5792, mae 0.0615, number of correct prediction 53304
+2024-07-06 03:18:29,323 - evaluation dataset 1 - steps 2512 - time 177.92s, valid loss 0.59908577
+2024-07-06 03:18:29,324 - 	valid stability spearman p 0.7392, mae 0.053, number of correct prediction 2512
+2024-07-06 03:44:15,622 - epoch 21/48 - steps 105, - time 1545.85s, train loss 0.14841781
+2024-07-06 03:44:15,622 - 	train stability spearman p 0.5938, mae 0.0597, number of correct prediction 53301
+2024-07-06 03:47:11,424 - evaluation dataset 1 - steps 2512 - time 175.77s, valid loss 0.59845865
+2024-07-06 03:47:11,424 - 	valid stability spearman p 0.7318, mae 0.0547, number of correct prediction 2512
+2024-07-06 04:12:55,730 - epoch 22/48 - steps 105, - time 1543.86s, train loss 0.14786205
+2024-07-06 04:12:55,730 - 	train stability spearman p 0.5975, mae 0.0592, number of correct prediction 53318
+2024-07-06 04:15:51,892 - evaluation dataset 1 - steps 2512 - time 176.13s, valid loss 0.61498887
+2024-07-06 04:15:51,892 - 	valid stability spearman p 0.7301, mae 0.0541, number of correct prediction 2512
+2024-07-06 04:41:36,075 - epoch 23/48 - steps 105, - time 1543.73s, train loss 0.14677871
+2024-07-06 04:41:36,075 - 	train stability spearman p 0.6106, mae 0.058, number of correct prediction 53307
+2024-07-06 04:44:28,976 - evaluation dataset 1 - steps 2512 - time 172.87s, valid loss 0.6000444
+2024-07-06 04:44:28,976 - 	valid stability spearman p 0.7346, mae 0.0541, number of correct prediction 2512
+2024-07-06 05:10:14,474 - epoch 24/48 - steps 105, - time 1545.04s, train loss 0.14593586
+2024-07-06 05:10:14,475 - 	train stability spearman p 0.6187, mae 0.0572, number of correct prediction 53306
+2024-07-06 05:13:12,427 - evaluation dataset 1 - steps 2512 - time 177.92s, valid loss 0.61570224
+2024-07-06 05:13:12,427 - 	valid stability spearman p 0.7339, mae 0.0533, number of correct prediction 2512
+2024-07-06 05:38:57,006 - epoch 25/48 - steps 105, - time 1544.13s, train loss 0.14512503
+2024-07-06 05:38:57,006 - 	train stability spearman p 0.623, mae 0.0566, number of correct prediction 53299
+2024-07-06 05:41:52,087 - evaluation dataset 1 - steps 2512 - time 175.05s, valid loss 0.63505717
+2024-07-06 05:41:52,088 - 	valid stability spearman p 0.7261, mae 0.056, number of correct prediction 2512
+2024-07-06 06:07:38,515 - epoch 26/48 - steps 105, - time 1545.97s, train loss 0.144289
+2024-07-06 06:07:38,515 - 	train stability spearman p 0.6285, mae 0.0559, number of correct prediction 53289
+2024-07-06 06:10:33,061 - evaluation dataset 1 - steps 2512 - time 174.51s, valid loss 0.61991238
+2024-07-06 06:10:33,061 - 	valid stability spearman p 0.731, mae 0.054, number of correct prediction 2512
+2024-07-06 06:36:17,122 - epoch 27/48 - steps 105, - time 1543.61s, train loss 0.14366293
+2024-07-06 06:36:17,122 - 	train stability spearman p 0.638, mae 0.0552, number of correct prediction 53314
+2024-07-06 06:39:15,832 - evaluation dataset 1 - steps 2512 - time 178.67s, valid loss 0.61764782
+2024-07-06 06:39:15,832 - 	valid stability spearman p 0.7258, mae 0.0547, number of correct prediction 2512
+2024-07-06 07:05:03,636 - epoch 28/48 - steps 105, - time 1547.35s, train loss 0.14253053
+2024-07-06 07:05:03,637 - 	train stability spearman p 0.6508, mae 0.0536, number of correct prediction 53320
+2024-07-06 07:08:02,321 - evaluation dataset 1 - steps 2512 - time 178.65s, valid loss 0.6193421
+2024-07-06 07:08:02,321 - 	valid stability spearman p 0.7204, mae 0.0551, number of correct prediction 2512
+2024-07-06 07:33:51,510 - epoch 29/48 - steps 105, - time 1548.74s, train loss 0.14204362
+2024-07-06 07:33:51,511 - 	train stability spearman p 0.6557, mae 0.0532, number of correct prediction 53301
+2024-07-06 07:36:47,359 - evaluation dataset 1 - steps 2512 - time 175.81s, valid loss 0.62733793
+2024-07-06 07:36:47,360 - 	valid stability spearman p 0.7292, mae 0.054, number of correct prediction 2512
+2024-07-06 08:02:33,704 - epoch 30/48 - steps 105, - time 1545.89s, train loss 0.14151941
+2024-07-06 08:02:33,705 - 	train stability spearman p 0.6556, mae 0.0532, number of correct prediction 53318
+2024-07-06 08:05:31,549 - evaluation dataset 1 - steps 2512 - time 177.8s, valid loss 0.63140021
+2024-07-06 08:05:31,550 - 	valid stability spearman p 0.73, mae 0.0545, number of correct prediction 2512
+2024-07-06 08:31:12,091 - epoch 31/48 - steps 105, - time 1540.08s, train loss 0.14072536
+2024-07-06 08:31:12,092 - 	train stability spearman p 0.6657, mae 0.0521, number of correct prediction 53324
+2024-07-06 08:34:08,121 - evaluation dataset 1 - steps 2512 - time 175.99s, valid loss 0.62926954
+2024-07-06 08:34:08,121 - 	valid stability spearman p 0.7225, mae 0.0551, number of correct prediction 2512
+2024-07-06 08:59:46,194 - epoch 32/48 - steps 105, - time 1537.62s, train loss 0.14034777
+2024-07-06 08:59:46,194 - 	train stability spearman p 0.6695, mae 0.0517, number of correct prediction 53295
+2024-07-06 09:02:43,284 - evaluation dataset 1 - steps 2512 - time 177.05s, valid loss 0.62579863
+2024-07-06 09:02:43,285 - 	valid stability spearman p 0.7206, mae 0.0555, number of correct prediction 2512
+2024-07-06 09:28:22,049 - epoch 33/48 - steps 105, - time 1538.3s, train loss 0.13980339
+2024-07-06 09:28:22,050 - 	train stability spearman p 0.6737, mae 0.0513, number of correct prediction 53305
+2024-07-06 09:31:15,518 - evaluation dataset 1 - steps 2512 - time 173.43s, valid loss 0.63753542
+2024-07-06 09:31:15,518 - 	valid stability spearman p 0.7256, mae 0.0544, number of correct prediction 2512
+2024-07-06 09:56:52,535 - epoch 34/48 - steps 105, - time 1536.56s, train loss 0.13909172
+2024-07-06 09:56:52,535 - 	train stability spearman p 0.6766, mae 0.0506, number of correct prediction 53312
+2024-07-06 09:59:46,873 - evaluation dataset 1 - steps 2512 - time 174.3s, valid loss 0.63789942
+2024-07-06 09:59:46,873 - 	valid stability spearman p 0.7275, mae 0.0545, number of correct prediction 2512
+2024-07-06 10:25:27,569 - epoch 35/48 - steps 105, - time 1540.24s, train loss 0.13869807
+2024-07-06 10:25:27,569 - 	train stability spearman p 0.6791, mae 0.0503, number of correct prediction 53313
+2024-07-06 10:28:22,042 - evaluation dataset 1 - steps 2512 - time 174.43s, valid loss 0.6326882
+2024-07-06 10:28:22,043 - 	valid stability spearman p 0.7234, mae 0.0548, number of correct prediction 2512
+2024-07-06 10:54:00,461 - epoch 36/48 - steps 105, - time 1537.96s, train loss 0.13812206
+2024-07-06 10:54:00,462 - 	train stability spearman p 0.6831, mae 0.0501, number of correct prediction 53320
+2024-07-06 10:56:55,864 - evaluation dataset 1 - steps 2512 - time 175.36s, valid loss 0.64251012
+2024-07-06 10:56:55,864 - 	valid stability spearman p 0.7197, mae 0.0548, number of correct prediction 2512
+2024-07-06 11:22:35,719 - epoch 37/48 - steps 105, - time 1539.39s, train loss 0.13756861
+2024-07-06 11:22:35,719 - 	train stability spearman p 0.6922, mae 0.0491, number of correct prediction 53355
+2024-07-06 11:25:31,957 - evaluation dataset 1 - steps 2512 - time 176.2s, valid loss 0.64436017
+2024-07-06 11:25:31,957 - 	valid stability spearman p 0.7115, mae 0.0561, number of correct prediction 2512
+2024-07-06 11:51:10,099 - epoch 38/48 - steps 105, - time 1537.69s, train loss 0.13751318
+2024-07-06 11:51:10,100 - 	train stability spearman p 0.686, mae 0.0495, number of correct prediction 53306
+2024-07-06 11:54:04,048 - evaluation dataset 1 - steps 2512 - time 173.91s, valid loss 0.64465082
+2024-07-06 11:54:04,049 - 	valid stability spearman p 0.7256, mae 0.0546, number of correct prediction 2512
+2024-07-06 12:19:41,515 - epoch 39/48 - steps 105, - time 1537.02s, train loss 0.13719218
+2024-07-06 12:19:41,515 - 	train stability spearman p 0.6877, mae 0.0492, number of correct prediction 53342
+2024-07-06 12:22:36,967 - evaluation dataset 1 - steps 2512 - time 175.41s, valid loss 0.65021068
+2024-07-06 12:22:36,968 - 	valid stability spearman p 0.7238, mae 0.055, number of correct prediction 2512
+2024-07-06 12:48:17,724 - epoch 40/48 - steps 105, - time 1540.3s, train loss 0.13661633
+2024-07-06 12:48:17,724 - 	train stability spearman p 0.6949, mae 0.0486, number of correct prediction 53314
+2024-07-06 12:51:12,931 - evaluation dataset 1 - steps 2512 - time 175.17s, valid loss 0.652825
+2024-07-06 12:51:12,931 - 	valid stability spearman p 0.7211, mae 0.0547, number of correct prediction 2512
+2024-07-06 13:16:53,360 - epoch 41/48 - steps 105, - time 1539.97s, train loss 0.13669245
+2024-07-06 13:16:53,360 - 	train stability spearman p 0.6922, mae 0.0489, number of correct prediction 53324
+2024-07-06 13:19:48,674 - evaluation dataset 1 - steps 2512 - time 175.27s, valid loss 0.64914848
+2024-07-06 13:19:48,675 - 	valid stability spearman p 0.7211, mae 0.055, number of correct prediction 2512
+2024-07-06 13:45:29,880 - epoch 42/48 - steps 105, - time 1540.73s, train loss 0.13632903
+2024-07-06 13:45:29,880 - 	train stability spearman p 0.6963, mae 0.0485, number of correct prediction 53351
+2024-07-06 13:48:25,810 - evaluation dataset 1 - steps 2512 - time 175.89s, valid loss 0.64850231
+2024-07-06 13:48:25,810 - 	valid stability spearman p 0.7226, mae 0.0547, number of correct prediction 2512
+2024-07-06 14:14:02,505 - epoch 43/48 - steps 105, - time 1536.22s, train loss 0.13618307
+2024-07-06 14:14:02,505 - 	train stability spearman p 0.6991, mae 0.0483, number of correct prediction 53298
+2024-07-06 14:16:58,703 - evaluation dataset 1 - steps 2512 - time 176.16s, valid loss 0.65442398
+2024-07-06 14:16:58,704 - 	valid stability spearman p 0.7183, mae 0.055, number of correct prediction 2512
+2024-07-06 14:42:39,325 - epoch 44/48 - steps 105, - time 1540.16s, train loss 0.1359691
+2024-07-06 14:42:39,325 - 	train stability spearman p 0.6957, mae 0.0484, number of correct prediction 53315
+2024-07-06 14:45:35,365 - evaluation dataset 1 - steps 2512 - time 176.0s, valid loss 0.65047561
+2024-07-06 14:45:35,365 - 	valid stability spearman p 0.7233, mae 0.0547, number of correct prediction 2512
+2024-07-06 15:11:12,465 - epoch 45/48 - steps 105, - time 1536.65s, train loss 0.13577146
+2024-07-06 15:11:12,465 - 	train stability spearman p 0.7031, mae 0.0477, number of correct prediction 53323
+2024-07-06 15:14:07,831 - evaluation dataset 1 - steps 2512 - time 175.32s, valid loss 0.6487082
+2024-07-06 15:14:07,832 - 	valid stability spearman p 0.721, mae 0.0548, number of correct prediction 2512
+2024-07-06 15:39:46,841 - epoch 46/48 - steps 105, - time 1538.55s, train loss 0.13574993
+2024-07-06 15:39:46,841 - 	train stability spearman p 0.7003, mae 0.0479, number of correct prediction 53324
+2024-07-06 15:42:41,823 - evaluation dataset 1 - steps 2512 - time 174.94s, valid loss 0.64949482
+2024-07-06 15:42:41,823 - 	valid stability spearman p 0.7194, mae 0.055, number of correct prediction 2512
+2024-07-06 16:08:19,528 - epoch 47/48 - steps 105, - time 1537.26s, train loss 0.13570484
+2024-07-06 16:08:19,528 - 	train stability spearman p 0.701, mae 0.0478, number of correct prediction 53306
+2024-07-06 16:11:13,899 - evaluation dataset 1 - steps 2512 - time 174.33s, valid loss 0.6503134
+2024-07-06 16:11:13,900 - 	valid stability spearman p 0.7194, mae 0.0549, number of correct prediction 2512
+2024-07-06 16:36:53,253 - epoch 48/48 - steps 105, - time 1538.82s, train loss 0.13576934
+2024-07-06 16:36:53,253 - 	train stability spearman p 0.7006, mae 0.0479, number of correct prediction 53315
+2024-07-06 16:39:47,753 - evaluation dataset 1 - steps 2512 - time 174.46s, valid loss 0.65165197
+2024-07-06 16:39:47,753 - 	valid stability spearman p 0.7164, mae 0.0551, number of correct prediction 2512
+2024-07-06 16:40:07,820 - 
+
+start testing the best validation checkpoints
+2024-07-06 16:40:07,820 - 
+	testing this checkpoint: /mnt/hdd8/mehdi/projects/prot2token/results/stability/2024-07-05__17-35-31/checkpoints/best_valid_stability_spearman.pth
+2024-07-06 16:40:25,836 - Loading checkpoint log: <All keys matched successfully>
+2024-07-06 16:55:53,205 - evaluation dataset 1 - steps 12851 - time 927.32s, valid loss 0.5607418
+2024-07-06 16:55:53,205 - 	test stability spearman p 0.7974, mae 0.0572, number of correct prediction 12851