Training in progress, step 18580

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3ead06ca64c097b468e4848a48dfe313e10e066882daadec4733b392f7d027f
+oid sha256:97e8843eb054c3ad74976f86442c8477d9387361a93f5b5aa761f197a5bc9063
 size 13982248

runs/Apr12_11-58-09_f044666cc877/events.out.tfevents.1712923163.f044666cc877.110.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d903f74c6c3b2a17c0879b3a1cec6a2056893f8043ad1d9d0ebbd7ffc341750d
-size 201502
+oid sha256:9beca122599c177ea752ce3b4320cde847f76d9c8733a1ab24ed8ad19c4110e7
+size 201717

wandb/debug-internal.log

@@ -13892,3 +13892,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:06,124 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:06,125 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:06,878 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:07,878 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:10,950 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:12,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:16,626 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:21,628 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:22,997 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:26:26,998 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:27,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:27,640 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:26:27,642 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:26:27,885 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

wandb/run-20240412_115951-o0z5y86z/files/output.log

@@ -1858,4 +1858,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5451, "train/grad_norm": 28.641721725463867, "train/learning_rate": 6.766473629355452e-07, "train/epoch": 0.26, "train/global_step": 18560, "_timestamp": 1712942766.1225753, "_runtime": 19574.531605243683, "_step": 927}
+{"train/loss": 3.5458, "train/grad_norm": 19.47591209411621, "train/learning_rate": 6.580644556884702e-07, "train/epoch": 0.26, "train/global_step": 18580, "_timestamp": 1712942787.6394272, "_runtime": 19596.04845714569, "_step": 928}

wandb/run-20240412_115951-o0z5y86z/logs/debug-internal.log

@@ -13892,3 +13892,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:06,124 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:06,125 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:06,878 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:07,878 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:10,950 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:12,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:16,626 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:21,628 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:22,997 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:26:26,998 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:27,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:27,640 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:26:27,642 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:26:27,885 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

wandb/run-20240412_115951-o0z5y86z/run-o0z5y86z.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8ef47c775ca6318a932a149b43d758f888d75599b45e13d35bbc99f4a5276ba
-size 1303574
+oid sha256:14f25204bb53d2ea3d718c32ff6e34e885fdb002a5de4ef2c0f53b3686676d7a
+size 1307440