Training in progress, step 3000

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:724b47d9c8653ca30a08e92b5b2db184a6a2691e07d099d47dd64cd6bfb9234f
+oid sha256:bf840ad1b562e045bc92ac31e01aee8f4cd67d1a73eca73333298edfbae1ad20
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db935a7bb28cf8189f92e0186d1529a18ea785c99045f42a546e5d1e7d565625
-size 36697
+oid sha256:116c85876965f9c37559552b22dbc758e5d81ed2a8908ff3c35d6d91dbd5f2c4
+size 36908

wandb/debug-internal.log

@@ -2501,3 +2501,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:41:36,997 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:41:37,440 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:41:40,441 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:41:40,709 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:45,153 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:41:46,154 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:46,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:46,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:46,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:51,832 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:56,832 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:01,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:01,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:01,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:01,853 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:02,723 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:02,730 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:03,451 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -294,4 +294,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5125, "train/grad_norm": 47.12533187866211, "train/learning_rate": 2.8899655018846297e-05, "train/epoch": 0.24, "train/global_step": 2980, "_timestamp": 1712911296.9946177, "_runtime": 3652.486899614334, "_step": 148}
+{"train/loss": 3.5043, "train/grad_norm": 23.667312622070312, "train/learning_rate": 2.8881318444640564e-05, "train/epoch": 0.24, "train/global_step": 3000, "_timestamp": 1712911322.7230785, "_runtime": 3678.215360403061, "_step": 149}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -2501,3 +2501,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:41:36,996 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 08:41:36,997 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:41:37,440 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 08:41:40,441 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:41:40,709 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:45,153 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 08:41:46,154 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:46,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:46,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:46,781 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:51,832 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:56,832 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:01,766 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:42:01,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:42:01,807 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:42:01,853 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:42:02,723 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:42:02,729 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:42:02,730 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:42:03,451 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json