Training in progress, step 60

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8873e4fdd5279928a04e9a0a09c50f9664887f0cb51b7ea9edff60bc99b21365
+oid sha256:de52612f18d94eb85267cdbaa87cfac914b5a3415370220f048ff6627b5dc973
 size 8077608

runs/Apr12_07-33-49_e5a48bec8248/events.out.tfevents.1712907230.e5a48bec8248.174.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c42462f9ce2b5c465f011bc8dbbfaa053a2fc677d11434902c64e9be8fa49e19
-size 5693
+oid sha256:c21610610a8dc52be98a16abd046244b6b488ecc1cbe5f7b64d018f3ccfa8961
+size 6248

wandb/debug-internal.log

@@ -283,3 +283,27 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 07:34:04,114 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 07:34:04,114 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 07:34:04,178 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/wandb-summary.json
+2024-04-12 07:34:06,179 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/output.log
+2024-04-12 07:34:08,197 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:34:08,198 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:34:08,201 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:34:08,317 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:34:10,454 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:34:10,455 DEBUG   SenderThread:216 [sender.py:send():379] send: history
+2024-04-12 07:34:10,455 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:10,456 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,182 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,184 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:34:11,185 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,185 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,186 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,187 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,187 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/wandb-summary.json
+2024-04-12 07:34:11,188 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,188 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,188 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,189 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send():379] send: history
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,190 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end

wandb/run-20240412_072736-gyl3aena/files/output.log

@@ -10,4 +10,6 @@ dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_072736-gyl3aena/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 4.5875, "train/grad_norm": 226.922119140625, "train/learning_rate": 7.97387338950315e-06, "train/epoch": 0.0, "train/global_step": 40, "_timestamp": 1712907244.1121442, "_runtime": 387.48006534576416, "_step": 5, "train_runtime": 39.0078, "train_samples_per_second": 3.076, "train_steps_per_second": 1.538, "total_flos": 19812939669504.0, "train_loss": 4.6811678568522135}
+{"train/loss": 4.8834, "train/grad_norm": 86.45769500732422, "train/learning_rate": 0.0, "train/epoch": 0.0, "train/global_step": 60, "_timestamp": 1712907251.181722, "_runtime": 394.54964303970337, "_step": 7, "train_runtime": 20.5872, "train_samples_per_second": 5.829, "train_steps_per_second": 2.914, "total_flos": 19812939669504.0, "train_loss": 4.715130106608073}

wandb/run-20240412_072736-gyl3aena/logs/debug-internal.log

@@ -283,3 +283,27 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 07:34:04,114 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 07:34:04,114 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 07:34:04,178 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/wandb-summary.json
+2024-04-12 07:34:06,179 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/output.log
+2024-04-12 07:34:08,197 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:34:08,198 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:34:08,201 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:34:08,317 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:34:10,454 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:34:10,455 DEBUG   SenderThread:216 [sender.py:send():379] send: history
+2024-04-12 07:34:10,455 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:10,456 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,182 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,184 DEBUG   HandlerThread:216 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:34:11,185 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,185 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,186 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,187 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,187 INFO    Thread-12 :216 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_072736-gyl3aena/files/wandb-summary.json
+2024-04-12 07:34:11,188 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,188 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,188 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,189 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send():379] send: history
+2024-04-12 07:34:11,189 DEBUG   SenderThread:216 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:34:11,190 INFO    SenderThread:216 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end