Training in progress, step 6460

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7d16dbb8cb7ed6096db49103af6741685f3a84a34f7cb454da74b440798e0a1
+oid sha256:4edb15d0e1b52d3bdcfb17044d4d5c2d1bc11a9b89494c965b286c71d0b836fb
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:006e9daa91992c399d22b4d31560e01139a6f83a6dc97f2011c1afc2a179348d
-size 72989
+oid sha256:286433de40d39036e464c8b32063577841d4a1cec0d0a46596bbfe206ceb73a3
+size 73411

wandb/debug-internal.log

@@ -5312,3 +5312,36 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:51:35,441 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:51:35,441 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:51:36,164 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:51:39,165 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:51:39,238 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:44,239 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:45,299 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:51:47,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:51:47,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:51:47,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:51:49,511 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:54,512 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:58,362 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:51:58,363 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:51:58,363 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:51:58,364 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:51:59,173 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:51:59,881 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:01,174 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:52:02,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:52:02,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:52:02,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:52:05,469 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:10,470 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:15,301 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:52:16,302 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:17,137 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:52:17,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:52:17,411 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:52:21,192 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:52:21,193 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:52:21,193 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:52:21,194 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:52:21,959 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:22,184 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:52:23,185 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -638,4 +638,10 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.4446, "train/grad_norm": 30.199399948120117, "train/learning_rate": 2.3816778784387097e-05, "train/epoch": 0.51, "train/global_step": 6420, "_timestamp": 1712915495.4393005, "_runtime": 7850.931582450867, "_step": 320}
+{"train/loss": 3.422, "train/grad_norm": 34.33303451538086, "train/learning_rate": 2.3737988124226834e-05, "train/epoch": 0.52, "train/global_step": 6460, "_timestamp": 1712915541.191715, "_runtime": 7896.683996915817, "_step": 322}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -5312,3 +5312,36 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:51:35,441 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:51:35,441 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:51:36,164 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:51:39,165 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:51:39,238 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:44,239 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:45,299 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:51:47,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:51:47,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:51:47,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:51:49,511 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:54,512 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:58,362 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:51:58,363 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:51:58,363 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:51:58,364 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:51:59,173 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:51:59,881 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:01,174 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:52:02,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:52:02,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:52:02,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:52:05,469 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:10,470 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:15,301 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:52:16,302 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:17,137 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:52:17,411 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:52:17,411 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:52:21,192 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:52:21,193 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:52:21,193 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:52:21,194 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:52:21,959 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:52:22,184 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:52:23,185 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log