Training in progress, step 3020

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf840ad1b562e045bc92ac31e01aee8f4cd67d1a73eca73333298edfbae1ad20
+oid sha256:5aafd43dd870cf48d59a15fc9271b5b10137c4bd726ab3b83b5240ec30b426d0
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:116c85876965f9c37559552b22dbc758e5d81ed2a8908ff3c35d6d91dbd5f2c4
-size 36908
+oid sha256:09042738286089b2d999c542068f99c6cab514aa64e22dbda8e22e4f177d82af
+size 37119

wandb/debug-internal.log

@@ -2519,3 +2519,18 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:42:02,730 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:42:03,451 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:42:06,452 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:42:07,451 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:12,452 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:15,154 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:42:16,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:16,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:16,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:17,872 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:22,873 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:26,718 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:26,719 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:26,719 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:26,720 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:27,460 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:42:28,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -296,4 +296,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5043, "train/grad_norm": 23.667312622070312, "train/learning_rate": 2.8881318444640564e-05, "train/epoch": 0.24, "train/global_step": 3000, "_timestamp": 1712911322.7230785, "_runtime": 3678.215360403061, "_step": 149}
+{"train/loss": 3.6236, "train/grad_norm": 26.047521591186523, "train/learning_rate": 2.8862836261972873e-05, "train/epoch": 0.24, "train/global_step": 3020, "_timestamp": 1712911346.7173917, "_runtime": 3702.209673643112, "_step": 150}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -2519,3 +2519,18 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:42:02,730 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:42:03,451 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:42:06,452 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:42:07,451 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:12,452 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:15,154 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:42:16,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:16,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:16,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:17,872 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:22,873 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:26,718 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:26,719 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:26,719 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:26,720 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:27,460 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:42:28,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report