Training in progress, step 2980

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38f3ea2e710f0123cd317966daf5b272ed615e4c14e337828c2d49a95e7b8e06
+oid sha256:724b47d9c8653ca30a08e92b5b2db184a6a2691e07d099d47dd64cd6bfb9234f
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de40781eb00de5bdf8551d130aa37f13f98e3fef388c451997820a31ec09baf3
-size 36486
+oid sha256:db935a7bb28cf8189f92e0186d1529a18ea785c99045f42a546e5d1e7d565625
+size 36697

wandb/debug-internal.log

@@ -2484,3 +2484,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:41:13,254 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:41:13,430 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:41:15,152 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:41:16,431 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:41:16,765 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:16,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:16,780 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:18,886 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:23,887 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:28,887 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:31,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:31,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:31,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:34,825 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:36,995 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:41:36,997 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:41:37,440 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -292,4 +292,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5737, "train/grad_norm": 24.437808990478516, "train/learning_rate": 2.8917845792248085e-05, "train/epoch": 0.24, "train/global_step": 2960, "_timestamp": 1712911273.251721, "_runtime": 3628.7440028190613, "_step": 147}
+{"train/loss": 3.5125, "train/grad_norm": 47.12533187866211, "train/learning_rate": 2.8899655018846297e-05, "train/epoch": 0.24, "train/global_step": 2980, "_timestamp": 1712911296.9946177, "_runtime": 3652.486899614334, "_step": 148}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -2484,3 +2484,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:41:13,254 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:41:13,430 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:41:15,152 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:41:16,431 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:41:16,765 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:16,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:16,780 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:18,886 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:23,887 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:28,887 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:31,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:31,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:31,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:34,825 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:36,995 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:41:36,997 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:41:37,440 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json