Training in progress, step 3040

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aafd43dd870cf48d59a15fc9271b5b10137c4bd726ab3b83b5240ec30b426d0
+oid sha256:2034464fd2bf2d3e12a454ef4d88017b36c93fe78dfe426fb33fb484b6010ab7
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09042738286089b2d999c542068f99c6cab514aa64e22dbda8e22e4f177d82af
-size 37119
+oid sha256:92eca165ef383c7ce15fcddba13159c7ef1920a4fcd7b82c9b347927229efa7e
+size 37330

wandb/debug-internal.log

@@ -2534,3 +2534,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:42:26,720 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:42:27,460 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 08:42:28,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:30,461 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:42:31,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:31,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:31,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:33,809 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:38,810 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:43,811 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:45,155 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:42:46,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:46,767 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:46,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:48,890 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:53,891 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:56,241 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:56,242 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:56,242 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:56,243 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:56,472 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -298,4 +298,8 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.6236, "train/grad_norm": 26.047521591186523, "train/learning_rate": 2.8862836261972873e-05, "train/epoch": 0.24, "train/global_step": 3020, "_timestamp": 1712911346.7173917, "_runtime": 3702.209673643112, "_step": 150}
+{"train/loss": 3.4851, "train/grad_norm": 39.91059494018555, "train/learning_rate": 2.8844208664712577e-05, "train/epoch": 0.24, "train/global_step": 3040, "_timestamp": 1712911376.240679, "_runtime": 3731.7329609394073, "_step": 151}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -2534,3 +2534,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:42:26,720 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:42:27,460 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 08:42:28,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:30,461 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:42:31,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:31,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:31,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:33,809 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:38,810 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:43,811 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:45,155 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:42:46,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:46,767 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:46,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:48,890 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:53,891 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:56,241 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:56,242 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:56,242 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:56,243 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:56,472 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json