Training in progress, step 9040

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cfa40939228184cf2cd9a1239f3986ffad3f7127a60ebded3c5b47c8a001f443
+oid sha256:ba139240646f433f9f0106d855aea9e350f07f031693b316743dd388f006a45a
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1545f216a17eb23cfc5eb03468610652c7d9554053cb35aeb59674b3f152ac8
-size 100208
+oid sha256:d2f229b0ff58ad3c7398f80c3698a6cf6357a6eab2eaddf535b2f272e8058080
+size 100630

wandb/debug-internal.log

@@ -7429,3 +7429,34 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:44:26,521 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 10:44:27,508 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 10:44:28,508 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:44:31,319 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:32,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:44:32,950 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:44:32,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:44:37,081 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:42,082 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:45,404 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:44:47,405 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:47,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:44:47,951 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:44:47,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:44:49,830 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:44:49,831 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:44:49,832 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:44:49,832 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:44:50,516 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:44:52,517 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:44:53,389 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:58,390 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:02,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:45:02,950 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:45:02,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:45:04,006 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:09,007 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:14,008 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:14,209 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:45:14,210 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:45:14,210 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:45:14,211 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:45:14,527 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:45:15,405 DEBUG   SenderThread:485 [sender.py:send():379] send: stats

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -898,4 +898,8 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.4905, "train/grad_norm": 30.022031784057617, "train/learning_rate": 1.813451344546913e-05, "train/epoch": 0.72, "train/global_step": 9000, "_timestamp": 1712918666.5190961, "_runtime": 11022.01137804985, "_step": 449}
+{"train/loss": 3.5014, "train/grad_norm": 26.23653793334961, "train/learning_rate": 1.8039430793163753e-05, "train/epoch": 0.72, "train/global_step": 9040, "_timestamp": 1712918714.2085314, "_runtime": 11069.700813293457, "_step": 451}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -7429,3 +7429,34 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:44:26,521 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 10:44:27,508 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 10:44:28,508 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:44:31,319 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:32,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:44:32,950 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:44:32,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:44:37,081 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:42,082 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:45,404 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:44:47,405 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:47,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:44:47,951 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:44:47,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:44:49,830 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:44:49,831 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:44:49,832 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:44:49,832 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:44:50,516 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:44:52,517 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:44:53,389 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:44:58,390 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:02,627 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:45:02,950 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:45:02,951 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:45:04,006 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:09,007 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:14,008 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:45:14,209 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:45:14,210 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:45:14,210 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:45:14,211 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:45:14,527 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:45:15,405 DEBUG   SenderThread:485 [sender.py:send():379] send: stats