diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -9155,3 +9155,1267 @@ 2024-01-22 20:53:27,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported 2024-01-22 20:53:29,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported 2024-01-22 20:53:29,796 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:31,389 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:31,820 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:31,832 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:32,071 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:33,857 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:33,875 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:34,118 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:53:35,391 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:36,247 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:53:36,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:36,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:37,119 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:38,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:38,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:39,392 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:40,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:53:40,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:40,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:40,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:53:40,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:53:42,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:42,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:42,656 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:43,393 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:44,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:44,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:46,498 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:46,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:47,394 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:48,545 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:48,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:48,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:50,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:50,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:51,396 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:52,640 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:52,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:54,330 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:54,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:54,701 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:55,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:53:55,397 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:53:55,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:53:55,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:53:56,729 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:56,741 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:58,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:58,784 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:59,398 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:00,204 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:00,819 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:00,834 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:02,858 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:02,871 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:03,399 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:04,120 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:54:04,895 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:04,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:05,973 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:07,287 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:54:07,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:07,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:07,400 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:09,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:09,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:10,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:54:10,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:54:10,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:54:11,402 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:11,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:11,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:11,790 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:13,403 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:13,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:13,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:15,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:15,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:17,404 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:17,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:17,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:17,661 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:19,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:19,582 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:21,406 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:21,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:21,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:23,576 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:23,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:23,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:25,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:54:25,407 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:25,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:54:25,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:54:25,700 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:25,716 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:27,746 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:27,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:29,334 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:29,409 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:29,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:29,809 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:31,838 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:31,853 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:33,410 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:33,884 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:33,896 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:34,121 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:54:35,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:35,927 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:35,942 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:37,412 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:38,333 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:54:38,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:38,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:40,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:54:40,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:40,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:40,558 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:40,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:54:40,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:54:41,413 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:42,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:42,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:44,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:44,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:45,415 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:45,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:46,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:46,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:48,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:48,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:50,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:50,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:50,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:52,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:52,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:53,417 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:54,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:54,673 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:55,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:54:55,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:54:55,568 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:54:56,699 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:56,713 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:56,714 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:54:57,418 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:54:58,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:54:58,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:00,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:00,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:01,420 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:02,247 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:02,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:02,826 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:04,123 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:55:04,849 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:04,862 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:05,422 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:06,888 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:06,900 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:08,071 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:09,276 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:55:09,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:09,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:09,423 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:10,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:55:10,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:55:10,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:55:11,013 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:55:11,015 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:55:11,015 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:55:11,015 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:55:11,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:11,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:11,423 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:55:12,423 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:13,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:13,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:13,424 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:14,016 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:15,424 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:15,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:15,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:17,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:17,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:19,426 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:19,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:19,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:19,776 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:21,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:21,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:23,427 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:23,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:23,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:25,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:55:25,559 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:25,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:55:25,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:55:25,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:25,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:27,428 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:27,677 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:27,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:29,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:29,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:31,429 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:31,449 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:31,760 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:31,771 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:33,798 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:33,810 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:34,124 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:55:35,431 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:35,838 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:35,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:37,322 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:37,882 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:37,896 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:39,432 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:40,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:55:40,343 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:55:40,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:40,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:40,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:55:40,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:55:42,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:42,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:42,489 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:43,434 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:44,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:44,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:46,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:46,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:47,435 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:48,127 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:48,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:48,523 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:49,436 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:50,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:50,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:52,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:52,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:53,283 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:55:53,436 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:54,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:54,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:55,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:55:55,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:55:55,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:55:56,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:56,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:57,437 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:55:58,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:58,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:55:59,170 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:00,745 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:00,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:01,438 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:02,783 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:02,794 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:04,126 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:56:04,820 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:04,833 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:05,128 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:05,440 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:06,860 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:06,875 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:08,903 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:08,917 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:09,441 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:10,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:56:10,559 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:10,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:56:10,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:56:11,301 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:56:11,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:11,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:13,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:13,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:13,442 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:15,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:15,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:15,666 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:17,443 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:17,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:17,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:19,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:19,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:21,444 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:21,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:21,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:21,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:23,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:23,582 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:25,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:56:25,444 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:25,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:56:25,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:56:25,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:25,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:26,792 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:27,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:27,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:29,445 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:29,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:29,715 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:31,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:31,763 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:32,251 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:33,446 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:33,793 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:33,804 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:34,127 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:56:35,836 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:35,848 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:37,447 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:37,882 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:37,895 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:38,126 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:39,921 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:39,933 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:40,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:56:40,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:56:40,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:56:41,449 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:42,319 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:56:42,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:42,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:43,449 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:43,891 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:44,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:44,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:46,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:46,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:46,803 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:56:46,805 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:56:46,806 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:56:46,806 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:56:47,450 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:47,451 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:56:48,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:48,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:49,451 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:49,807 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:50,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:50,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:51,451 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:52,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:52,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:54,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:54,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:55,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:56:55,452 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:56:55,559 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:56:55,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:56:55,569 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:56:56,628 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:56,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:58,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:58,684 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:56:59,453 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:00,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:00,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:01,504 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:02,748 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:02,760 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:03,454 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:04,129 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:57:04,784 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:04,800 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:06,824 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:06,835 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:07,292 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:07,455 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:08,858 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:08,870 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:10,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:57:10,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:57:10,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:57:10,896 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:10,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:11,456 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:12,710 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:13,284 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:57:13,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:13,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:15,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:15,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:15,457 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:17,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:17,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:17,998 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:19,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:19,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:19,458 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:21,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:21,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:23,459 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:23,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:23,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:23,880 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:25,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:57:25,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:25,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:57:25,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:57:25,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:27,460 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:27,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:27,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:29,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:29,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:29,681 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:31,461 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:31,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:31,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:33,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:33,742 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:34,130 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:57:35,131 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:35,462 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:35,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:35,781 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:37,810 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:37,821 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:39,463 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:39,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:39,865 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:40,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:57:40,454 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:40,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:57:40,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:57:41,896 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:41,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:43,464 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:44,294 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:57:44,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:44,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:46,252 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:46,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:46,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:47,465 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:48,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:48,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:50,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:50,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:51,466 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:52,129 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:52,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:52,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:53,467 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:54,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:54,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:55,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:57:55,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:57:55,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:57:56,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:56,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:57,468 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:57:57,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:57:58,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:57:58,622 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:00,649 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:00,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:01,469 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:02,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:02,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:03,855 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:04,132 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:58:04,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:04,751 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:05,470 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:06,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:06,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:08,816 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:08,828 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:09,471 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:09,620 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:10,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:58:10,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:58:10,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:58:10,854 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:10,868 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:12,894 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:12,910 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:13,472 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:15,288 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:58:15,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:15,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:15,545 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:17,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:17,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:17,473 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:19,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:19,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:21,373 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:21,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:21,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:21,474 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:23,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:23,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:24,307 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:58:24,308 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:58:24,309 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:58:24,309 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:58:24,475 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:58:25,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:58:25,475 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:25,522 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:25,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:25,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:58:25,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:58:26,719 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:27,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:27,579 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:29,476 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:29,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:29,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:31,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:31,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:32,031 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:33,477 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:33,695 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:33,707 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:34,133 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:58:35,733 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:35,744 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:37,478 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:37,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:37,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:37,948 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:39,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:39,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:40,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:58:40,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:58:40,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:58:41,479 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:41,870 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:41,882 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:43,729 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:43,910 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:43,922 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:45,480 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:46,299 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:58:46,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:46,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:48,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:48,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:49,481 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:49,610 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:50,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:50,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:52,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:52,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:53,482 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:54,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:54,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:55,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:58:55,389 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:58:55,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:58:55,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:58:56,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:56,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:57,483 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:58:58,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:58:58,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:00,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:00,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:01,295 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:01,484 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:02,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:02,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:03,485 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:04,135 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:59:04,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:04,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:06,739 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:06,751 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:07,065 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:07,486 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:08,779 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:08,790 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:10,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:59:10,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:59:10,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:59:10,813 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:10,827 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:11,486 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:12,851 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:12,865 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:12,962 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:14,893 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:14,911 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:15,487 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:17,283 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:59:17,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:17,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:18,730 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:19,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:19,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:19,490 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:21,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:21,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:23,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:23,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:24,491 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:24,608 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:25,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:59:25,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:25,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:25,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:59:25,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:59:27,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:27,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:28,492 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:29,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:29,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:30,445 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:31,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:31,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:32,493 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:33,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:33,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:34,136 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:59:35,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:35,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:36,367 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:36,494 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:37,704 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:37,715 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:39,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:39,760 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:40,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:59:40,494 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:40,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:59:40,570 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:59:41,758 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:41,790 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:41,801 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:43,833 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:43,845 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:44,495 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:45,870 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:45,886 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:47,089 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:48,260 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:59:48,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:48,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:48,496 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:50,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:50,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:52,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:52,386 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:52,497 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:52,979 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:59:54,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:54,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:55,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:59:55,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:59:55,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:59:56,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:56,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:56,498 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:59:58,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:58,498 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:59:58,794 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:00,499 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:00,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:00,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:01,725 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 21:00:01,727 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 21:00:01,727 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 21:00:01,728 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 21:00:02,500 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 21:00:02,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:02,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:03,500 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:04,138 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:00:04,139 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:04,500 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:04,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:04,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:06,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:06,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:08,501 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:08,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:08,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:09,481 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:10,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:00:10,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:00:10,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:00:10,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:10,733 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:12,502 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:12,759 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:12,771 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:14,802 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:14,813 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:15,363 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:16,503 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:16,845 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:16,857 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:18,504 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:19,232 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:00:19,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:19,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:21,162 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:21,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:21,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:22,505 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:23,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:23,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:25,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:00:25,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:25,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:25,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:00:25,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:00:26,506 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:26,789 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:27,422 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:27,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:29,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:29,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:30,507 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:31,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:31,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:31,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:33,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:33,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:34,139 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:00:34,508 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:35,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:35,592 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:37,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:37,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:37,744 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:38,509 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:39,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:39,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:40,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:00:40,570 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:00:40,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:00:41,697 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:41,708 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:42,510 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:42,777 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:43,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:43,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:45,781 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:45,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:46,511 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:47,817 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:47,829 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:48,381 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:50,201 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:00:50,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:50,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:50,511 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:52,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:52,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:54,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:00:54,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:54,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:54,512 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:00:55,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:00:55,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:00:55,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:00:56,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:56,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:58,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:58,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:00:58,513 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:00,058 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:00,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:00,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:02,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:02,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:02,514 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:04,141 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:01:04,523 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:04,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:05,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:06,515 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:06,561 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:06,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:08,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:08,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:10,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:01:10,516 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:10,561 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:10,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:01:10,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:01:10,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:10,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:12,681 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:12,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:14,517 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:14,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:14,729 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:15,637 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:16,756 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:16,768 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:18,518 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:18,796 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:18,813 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:21,186 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:01:21,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:21,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:21,535 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:22,519 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:23,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:23,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:25,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:01:25,291 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:25,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:25,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:01:25,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:01:26,521 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:26,735 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:27,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:27,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:28,521 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:29,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:29,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:31,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:31,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:32,203 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:32,522 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:33,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:33,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:34,142 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:01:35,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:35,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:36,523 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:37,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:37,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:38,112 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:39,045 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 21:01:39,047 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 21:01:39,047 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 21:01:39,048 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 21:01:39,524 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 21:01:39,600 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:39,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:40,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:01:40,524 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:40,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:01:40,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:01:41,524 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:41,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:41,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:43,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:43,689 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:43,890 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:44,525 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:45,715 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:45,727 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:47,755 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:47,769 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:48,526 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:49,797 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:49,807 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:49,808 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:52,186 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:01:52,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:52,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:52,527 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:54,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:54,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:55,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:01:55,561 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:01:55,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:01:55,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:01:56,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:56,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:56,528 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:01:58,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:01:58,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:00,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:00,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:01,282 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:02,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:02,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:02,530 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:04,144 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:02:04,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:04,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:04,530 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:06,359 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:06,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:06,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:08,531 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:08,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:08,586 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:10,276 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:02:10,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:02:10,571 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:02:10,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:10,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:11,800 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:12,532 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:12,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:12,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:14,692 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:14,704 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:16,533 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:16,727 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:16,746 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:17,157 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:18,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:18,786 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:20,534 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:20,811 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:20,823 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:22,946 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:23,201 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:02:23,224 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:23,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:24,535 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:25,277 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:02:25,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:25,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:25,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:02:25,572 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:02:27,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:27,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:28,537 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:28,828 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:29,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:29,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:31,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:31,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:32,538 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:33,474 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:33,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:34,146 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:02:34,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:35,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:35,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:36,539 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:37,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:37,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:39,464 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:39,579 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:39,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:40,277 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:02:40,540 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:40,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:02:40,572 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:02:41,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:41,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:42,541 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:43,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:43,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:45,204 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:45,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:45,699 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:46,542 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:47,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:47,764 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:49,784 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:49,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:50,543 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:51,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:51,818 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:51,829 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:54,326 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:02:54,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:54,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:54,545 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:02:55,277 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:02:55,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:02:55,572 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:02:56,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:56,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:56,750 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:02:58,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:58,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:02:58,546 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:00,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:00,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:02,025 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:02,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:02,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:02,547 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:04,147 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:03:04,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:04,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:06,548 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:06,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:06,628 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:07,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:08,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:08,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:10,277 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:03:10,549 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:10,572 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:03:10,572 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:03:10,692 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:10,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:12,728 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:12,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:13,656 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:14,468 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 21:03:14,469 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 21:03:14,471 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 21:03:14,471 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 21:03:14,471 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 21:03:14,472 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 21:03:14,473 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 21:03:14,550 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:14,550 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 21:03:14,764 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:14,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:15,550 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:16,551 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:16,798 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:16,811 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:18,835 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:18,904 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:19,474 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:20,928 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:20,940 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:22,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:22,975 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:24,474 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:25,352 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:03:25,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:25,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:27,235 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:03:27,236 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:03:27,238 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:03:27,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:27,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:29,477 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:29,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:29,522 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:31,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:31,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:32,555 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:33,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:33,871 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:34,507 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:03:34,508 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:35,894 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:35,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:37,932 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:37,944 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:39,509 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:39,968 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:39,979 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:42,004 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:42,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:43,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:03:43,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:03:43,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:03:44,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:44,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:45,372 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:46,562 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/config.yaml +2024-01-22 21:03:46,645 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:46,657 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:48,562 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:48,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:48,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:50,716 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:50,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:51,083 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:52,998 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:53,012 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:54,564 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 21:03:55,035 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:55,049 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:56,097 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:03:57,426 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:03:57,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:57,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:58,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:03:58,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:03:58,186 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:03:59,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:03:59,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:01,100 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:01,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:01,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:03,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:03,579 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:04,151 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:04:05,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:05,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:06,176 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:07,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:07,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:09,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:09,698 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:11,277 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:11,720 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:11,731 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:13,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:04:13,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:04:13,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:04:13,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:13,764 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:15,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:15,800 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:16,417 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:17,828 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:17,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:19,862 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:19,874 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:21,575 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:21,896 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:21,912 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:23,938 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:23,949 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:25,975 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:25,989 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:26,695 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:28,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:04:28,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:04:28,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:04:28,375 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:04:28,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:28,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:30,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:30,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:32,014 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:32,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:32,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:34,152 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:04:34,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:34,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:36,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:36,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:37,041 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:38,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:38,614 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:40,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:40,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:42,166 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:42,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:42,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:43,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:04:43,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:04:43,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:04:44,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:44,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:46,747 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:46,758 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:47,523 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:48,780 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:48,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:50,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:50,826 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:52,543 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:52,848 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:52,859 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:54,882 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:54,893 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:56,919 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:56,931 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:58,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:04:58,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:04:58,188 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:04:58,188 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:04:59,305 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:04:59,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:04:59,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:01,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:01,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:03,372 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:03,420 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:03,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:04,154 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:05:05,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:05,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:07,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:07,522 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:08,721 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:09,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:09,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:11,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:11,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:13,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:05:13,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:05:13,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:05:13,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:13,645 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:13,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:15,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:15,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:17,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:17,714 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:19,002 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:19,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:19,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:21,771 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:21,783 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:23,808 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:23,821 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:24,077 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:25,844 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:25,856 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:27,878 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:27,889 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:28,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:05:28,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:05:28,187 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:05:29,508 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:30,264 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:05:30,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:30,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:32,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:32,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:34,156 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 21:05:34,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:34,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:34,840 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:36,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:36,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:38,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:38,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:40,025 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:40,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:40,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:42,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:42,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:43,185 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:05:43,188 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:05:43,188 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:05:44,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:44,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:45,315 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:46,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:46,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:48,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:48,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:50,404 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:50,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:50,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:52,728 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:52,739 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:54,761 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:54,779 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:55,404 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:05:56,802 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:56,816 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:58,186 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 21:05:58,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 21:05:58,188 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 21:05:58,838 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:05:58,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:06:00,524 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 21:06:01,246 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 21:06:01,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:06:01,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:06:03,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:06:03,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 21:06:04,159 DEBUG SenderThread:123548 [sender.py:send():382] send: stats