Training in progress, step 8680

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:328a66cde612fe77867acd947336918db03cd97261bd08ba4f2fa9a443e928a5
+oid sha256:bf5c561f13ecd3856bca353010f8f2bf59f14e6499e5570d5c4f503283ebee79
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c6506e7cbcbeb58a24f6dfd844a07a58e52b29eeea56f3d29d902af3eb5da52
-size 96410
+oid sha256:6a83778be7128243fc086f3551194f4a9bf55f9e5b86e0f62f87306e8dc17434
+size 96832

wandb/debug-internal.log

@@ -7135,3 +7135,39 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:37:08,882 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 10:37:08,883 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 10:37:09,333 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:37:11,674 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:12,335 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:37:15,388 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:37:17,389 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:17,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:17,755 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:17,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:22,870 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:27,871 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:32,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:32,755 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:32,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:33,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:37:33,768 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:37:33,768 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:37:33,769 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:37:33,769 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:34,344 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:37:36,344 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:37:39,300 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:44,301 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:45,388 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:37:47,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:47,756 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:47,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:49,849 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:54,850 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:59,851 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:38:01,742 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:38:01,743 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:38:01,744 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:38:01,744 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:38:02,360 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:38:02,515 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:38:02,756 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:38:02,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -860,4 +860,10 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.4675, "train/grad_norm": 23.41849708557129, "train/learning_rate": 1.8983862171241577e-05, "train/epoch": 0.69, "train/global_step": 8640, "_timestamp": 1712918228.8806434, "_runtime": 10584.372925281525, "_step": 431}
+{"train/loss": 3.432, "train/grad_norm": 26.743480682373047, "train/learning_rate": 1.8890106104060177e-05, "train/epoch": 0.69, "train/global_step": 8680, "_timestamp": 1712918281.7418907, "_runtime": 10637.234172582626, "_step": 433}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -7135,3 +7135,39 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:37:08,882 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 10:37:08,883 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 10:37:09,333 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:37:11,674 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:12,335 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:37:15,388 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:37:17,389 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:17,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:17,755 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:17,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:22,870 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:27,871 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:32,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:32,755 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:32,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:33,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:37:33,768 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:37:33,768 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:37:33,769 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:37:33,769 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:34,344 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:37:36,344 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:37:39,300 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:44,301 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:45,388 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 10:37:47,407 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:37:47,756 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:37:47,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:37:49,849 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:54,850 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:37:59,851 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:38:01,742 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:38:01,743 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:38:01,744 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:38:01,744 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:38:02,360 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:38:02,515 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:38:02,756 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:38:02,756 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status