diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,1935 +1,1008 @@ -2024-02-01 17:38:28,445 INFO StreamThr :237521 [internal.py:wandb_internal():86] W&B internal server running at pid: 237521, started at: 2024-02-01 17:38:28.443368 -2024-02-01 17:38:28,446 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status -2024-02-01 17:38:28,454 INFO WriterThread:237521 [datastore.py:open_for_write():85] open: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/run-py26nu6m.wandb -2024-02-01 17:38:28,455 DEBUG SenderThread:237521 [sender.py:send():382] send: header -2024-02-01 17:38:28,476 DEBUG SenderThread:237521 [sender.py:send():382] send: run -2024-02-01 17:38:28,713 INFO SenderThread:237521 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files -2024-02-01 17:38:28,713 INFO SenderThread:237521 [sender.py:_start_run_threads():1136] run started: py26nu6m with start time 1706809108.451874 -2024-02-01 17:38:28,720 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: check_version -2024-02-01 17:38:28,720 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: check_version -2024-02-01 17:38:28,774 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: run_start -2024-02-01 17:38:28,833 DEBUG HandlerThread:237521 [system_info.py:__init__():32] System info init -2024-02-01 17:38:28,833 DEBUG HandlerThread:237521 [system_info.py:__init__():47] System info init done -2024-02-01 17:38:28,833 INFO HandlerThread:237521 [system_monitor.py:start():194] Starting system monitor -2024-02-01 17:38:28,833 INFO SystemMonitor:237521 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-01 17:38:28,833 INFO HandlerThread:237521 [system_monitor.py:probe():214] Collecting system info -2024-02-01 17:38:28,834 INFO SystemMonitor:237521 [interfaces.py:start():190] Started cpu monitoring -2024-02-01 17:38:28,834 INFO SystemMonitor:237521 [interfaces.py:start():190] Started disk monitoring -2024-02-01 17:38:28,834 INFO SystemMonitor:237521 [interfaces.py:start():190] Started gpu monitoring -2024-02-01 17:38:28,836 INFO SystemMonitor:237521 [interfaces.py:start():190] Started memory monitoring -2024-02-01 17:38:28,836 INFO SystemMonitor:237521 [interfaces.py:start():190] Started network monitoring -2024-02-01 17:38:28,880 DEBUG HandlerThread:237521 [system_info.py:probe():196] Probing system -2024-02-01 17:38:28,883 DEBUG HandlerThread:237521 [system_info.py:_probe_git():181] Probing git -2024-02-01 17:38:28,908 DEBUG HandlerThread:237521 [system_info.py:_probe_git():189] Probing git done -2024-02-01 17:38:28,908 DEBUG HandlerThread:237521 [system_info.py:probe():244] Probing system done -2024-02-01 17:38:28,908 DEBUG HandlerThread:237521 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.5', 'heartbeatAt': '2024-02-01T17:38:28.880477', 'startedAt': '2024-02-01T17:38:28.419493', 'docker': None, 'cuda': None, 'args': ('config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co./sanchit-gandhi/distil-zephyr-1.5b-ssft', 'commit': '79a4ae874a71e67016ded927e7d23351e5c7dab8'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft', 'host': 'ip-26-0-165-24', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2731.1230833333334, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.237, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3593.987, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3597.474, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3587.972, 'min': 0.0, 'max': 0.0}, {'current': 3597.373, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2899.882, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.404, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3597.582, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 57.44935989379883}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855346679688}} -2024-02-01 17:38:28,908 INFO HandlerThread:237521 [system_monitor.py:probe():224] Finished collecting system info -2024-02-01 17:38:28,908 INFO HandlerThread:237521 [system_monitor.py:probe():227] Publishing system info -2024-02-01 17:38:28,908 DEBUG HandlerThread:237521 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment -2024-02-01 17:38:28,910 DEBUG HandlerThread:237521 [system_info.py:_save_pip():68] Saving pip packages done -2024-02-01 17:38:28,910 DEBUG HandlerThread:237521 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment -2024-02-01 17:38:29,716 INFO Thread-12 :237521 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/conda-environment.yaml -2024-02-01 17:38:29,716 INFO Thread-12 :237521 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/requirements.txt -2024-02-01 17:38:34,420 DEBUG HandlerThread:237521 [system_info.py:_save_conda():87] Saving conda packages done -2024-02-01 17:38:34,423 INFO HandlerThread:237521 [system_monitor.py:probe():229] Finished publishing system info -2024-02-01 17:38:34,457 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:38:34,457 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: keepalive -2024-02-01 17:38:34,458 DEBUG SenderThread:237521 [sender.py:send():382] send: files -2024-02-01 17:38:34,458 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now -2024-02-01 17:38:34,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:38:34,465 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:38:34,466 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:38:34,553 DEBUG SenderThread:237521 [sender.py:send():382] send: telemetry -2024-02-01 17:38:34,554 DEBUG SenderThread:237521 [sender.py:send():382] send: config -2024-02-01 17:38:34,555 DEBUG SenderThread:237521 [sender.py:send():382] send: metric -2024-02-01 17:38:34,555 DEBUG SenderThread:237521 [sender.py:send():382] send: telemetry -2024-02-01 17:38:34,555 DEBUG SenderThread:237521 [sender.py:send():382] send: metric -2024-02-01 17:38:34,555 WARNING SenderThread:237521 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) -2024-02-01 17:38:34,555 DEBUG SenderThread:237521 [sender.py:send():382] send: telemetry -2024-02-01 17:38:34,721 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/conda-environment.yaml -2024-02-01 17:38:34,722 INFO Thread-12 :237521 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-metadata.json -2024-02-01 17:38:34,722 INFO Thread-12 :237521 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:34,796 INFO wandb-upload_0:237521 [upload_job.py:push():131] Uploaded file /tmp/tmpfs5f2n7fwandb/421ry27q-wandb-metadata.json -2024-02-01 17:38:36,724 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:39,559 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:38:42,110 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:38:42,112 DEBUG SenderThread:237521 [sender.py:send():382] send: metric -2024-02-01 17:38:42,112 DEBUG SenderThread:237521 [sender.py:send():382] send: metric -2024-02-01 17:38:42,112 DEBUG SenderThread:237521 [sender.py:send():382] send: metric -2024-02-01 17:38:42,112 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:38:42,113 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:38:42,116 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:38:42,732 INFO Thread-12 :237521 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:38:44,735 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:44,770 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:38:45,673 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:38:45,674 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:38:45,674 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:38:45,676 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:38:45,737 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:38:46,738 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:48,741 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:49,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:38:49,465 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:38:49,467 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:38:50,190 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:38:50,230 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:38:50,232 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:38:50,232 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:38:50,234 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:38:50,745 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:38:50,745 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:52,747 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:54,715 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:38:54,716 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:38:54,716 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:38:54,718 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:38:54,751 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:38:54,751 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:55,617 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:38:56,753 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:58,756 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:38:59,239 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:38:59,240 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:38:59,240 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:38:59,242 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:38:59,758 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:00,760 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:01,049 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:01,762 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/config.yaml -2024-02-01 17:39:02,763 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:03,754 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:03,755 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:03,755 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:03,757 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:03,766 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:04,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:39:04,466 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:39:04,466 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:39:04,767 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:06,478 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:06,770 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:08,293 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:08,294 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:08,295 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:08,297 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:08,773 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:08,774 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:10,776 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:11,923 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:12,779 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:12,828 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:12,830 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:12,830 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:12,832 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:13,781 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:14,782 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:16,785 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:17,363 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:17,365 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:17,366 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:17,367 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:17,368 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:17,788 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:18,789 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:19,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:39:19,466 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:39:19,467 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:39:20,792 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:21,909 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:21,911 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:21,911 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:21,913 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:22,795 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:22,796 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:22,815 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:24,798 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:26,448 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:26,450 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:26,450 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:26,452 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:26,802 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:26,802 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:28,269 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:28,805 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:28,836 DEBUG SystemMonitor:237521 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-01 17:39:28,850 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:39:30,807 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:31,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:31,003 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:31,003 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:31,005 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:31,810 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:32,811 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:33,729 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:34,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:39:34,465 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:39:34,468 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:39:34,814 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:35,548 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:35,550 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:35,550 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:35,552 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:35,816 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:36,817 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:38,820 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:39,188 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:40,104 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:40,105 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:40,105 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:40,108 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:40,824 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:40,824 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:42,826 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:44,651 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:44,652 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:44,653 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:44,653 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:44,655 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:44,830 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:44,830 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:46,833 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:48,835 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:49,211 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:49,212 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:49,212 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:49,214 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:49,466 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:39:49,466 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:39:49,467 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:39:49,838 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:50,121 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:50,839 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:52,842 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:53,762 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:53,763 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:53,763 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:53,765 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:53,844 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:54,845 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:55,580 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:39:56,848 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:39:58,314 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:39:58,316 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:39:58,316 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:39:58,318 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:39:58,842 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:39:58,852 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:39:58,852 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:00,854 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:01,035 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:02,853 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:02,855 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:02,855 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:02,857 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:02,858 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:02,859 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:04,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:40:04,466 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:40:04,467 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:40:04,861 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:06,498 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:06,864 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:07,408 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:07,409 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:07,409 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:07,411 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:07,866 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:08,867 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:10,870 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:11,953 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:11,954 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:11,955 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:11,956 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:11,958 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:12,874 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:12,874 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:14,876 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:16,879 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:17,215 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:18,882 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:19,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:40:19,959 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:40:19,959 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:40:22,278 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:22,888 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:24,892 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:26,895 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:27,568 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:27,569 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:27,571 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:27,571 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:27,573 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:27,897 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:28,844 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:40:28,898 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:30,901 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:32,115 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:32,117 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:32,117 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:32,119 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:32,904 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:32,904 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:33,026 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:34,907 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:34,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:40:34,920 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:40:34,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:40:36,665 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:36,667 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:36,667 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:36,669 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:36,910 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:36,911 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:38,487 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:38,913 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:40,915 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:41,219 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:41,220 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:41,221 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:41,223 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:41,917 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:42,919 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:43,949 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:44,922 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:45,773 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:45,775 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:45,776 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:45,778 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:45,924 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:46,925 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:48,927 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:49,410 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:49,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:40:49,920 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:40:49,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:40:50,362 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:50,930 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:51,403 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:51,403 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:51,405 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:51,932 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:52,934 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:54,873 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:40:54,874 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:54,876 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:54,876 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:54,878 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:54,937 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:40:54,937 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:56,940 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:58,846 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:40:58,942 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:40:59,420 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:40:59,422 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:40:59,422 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:40:59,424 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:40:59,944 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:00,330 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:00,946 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:02,948 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:03,975 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:03,976 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:03,976 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:03,978 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:04,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:41:04,920 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:41:04,952 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:04,952 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:04,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:41:05,797 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:06,955 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:08,527 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:08,529 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:08,529 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:08,531 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:08,958 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:08,959 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:10,961 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:11,264 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:12,964 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:13,085 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:13,086 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:13,087 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:13,089 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:13,966 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:14,967 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:16,736 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:16,969 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:17,649 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:17,650 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:17,651 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:17,653 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:17,972 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:18,973 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:19,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:41:19,920 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:41:19,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:41:20,976 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:22,234 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:22,235 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:22,236 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:22,236 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:22,238 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:22,979 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:22,979 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:24,981 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:26,803 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:26,804 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:26,805 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:26,806 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:26,985 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:26,985 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:27,718 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:28,848 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:41:28,987 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:30,990 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:31,370 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:31,371 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:31,371 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:31,373 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:31,992 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:32,993 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:33,202 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:34,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:41:34,920 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:41:34,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:41:34,996 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:35,982 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:35,984 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:35,984 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:35,986 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:35,998 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:36,999 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:38,688 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:39,002 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:40,512 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:40,513 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:40,513 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:40,515 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:41,005 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:41,006 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:43,008 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:44,171 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:45,011 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:45,090 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:45,091 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:45,092 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:45,093 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:46,013 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:47,014 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:49,017 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:49,653 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:49,654 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:49,655 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:49,656 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:49,658 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:49,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:41:49,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:41:49,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:41:50,019 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:51,020 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:53,023 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:54,225 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:41:54,226 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:41:54,227 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:41:54,228 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:41:55,026 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:41:55,026 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:55,229 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:41:57,029 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:41:58,859 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:42:00,860 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:01,035 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:04,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:42:04,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:42:04,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:42:05,040 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:06,567 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:07,042 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:11,047 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:11,821 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:13,050 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:13,639 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:13,641 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:13,641 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:13,643 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:14,052 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:15,053 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:17,056 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:17,301 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:18,210 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:18,212 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:18,212 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:18,214 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:19,059 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:19,059 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:19,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:42:19,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:42:19,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:42:21,062 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:22,795 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:22,796 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:22,797 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:22,797 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:22,800 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:23,065 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:23,065 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:25,068 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:27,070 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:27,347 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:27,348 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:27,349 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:27,350 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:28,072 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:28,262 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:28,852 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:42:29,074 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:31,076 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:31,920 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:31,921 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:31,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:31,923 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:32,078 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:33,080 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:33,747 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:34,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:42:34,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:42:34,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:42:35,082 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:36,493 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:36,494 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:36,495 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:36,496 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:37,086 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:37,086 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:39,088 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:39,242 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:41,072 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:41,073 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:41,074 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:41,075 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:41,092 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:41,092 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:43,094 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:44,723 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:45,097 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:45,640 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:45,642 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:45,642 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:45,644 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:46,099 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:47,100 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:49,103 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:49,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:42:49,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:42:49,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:42:50,002 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:50,259 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:50,260 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:50,260 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:50,262 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:51,107 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:51,107 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:53,109 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:54,791 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:54,793 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:54,793 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:54,795 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:42:55,113 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:42:55,113 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:55,708 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:42:57,115 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:58,863 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:42:59,118 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:42:59,371 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:42:59,372 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:42:59,373 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:42:59,374 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:00,120 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:01,121 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:01,196 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:03,124 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:03,937 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:03,938 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:03,939 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:03,941 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:04,126 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:04,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:43:04,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:43:04,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:43:05,127 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:06,691 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:07,130 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:08,525 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:08,526 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:08,527 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:08,529 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:09,133 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:09,134 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:11,136 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:12,188 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:13,107 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:13,109 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:13,109 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:13,111 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:13,140 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:13,140 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:15,143 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:17,145 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:17,675 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:17,676 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:17,678 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:17,678 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:17,680 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:18,147 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:19,149 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:19,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:43:19,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:43:19,957 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:43:21,151 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:22,250 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:22,251 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:22,252 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:22,253 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:23,155 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:23,155 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:23,164 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:25,157 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:26,819 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:26,820 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:26,821 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:26,823 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:27,161 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:27,161 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:28,649 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:28,856 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:43:29,164 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:31,166 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:31,392 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:31,394 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:31,394 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:31,396 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:32,169 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:32,169 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:34,139 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:34,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:43:34,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:43:34,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:43:35,172 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:36,014 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:36,016 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:36,016 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:36,018 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:36,175 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:37,176 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:38,177 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:39,634 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:40,551 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:40,552 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:40,552 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:40,554 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:43:41,182 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:43:41,182 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:42,183 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:44,713 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:46,190 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:49,818 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:49,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:43:49,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:43:49,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:43:50,196 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:53,199 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:55,675 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:43:57,204 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:58,858 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:43:59,206 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:43:59,738 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:43:59,739 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:43:59,739 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:43:59,742 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:00,209 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:01,210 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:01,563 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:02,211 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:04,300 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:04,301 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:04,301 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:04,303 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:04,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:44:04,921 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:44:04,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:44:05,216 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:05,216 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:06,217 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:07,030 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:08,853 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:08,854 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:08,854 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:08,856 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:09,222 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:09,222 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:10,223 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:12,507 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:13,227 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:13,417 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:13,419 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:13,419 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:13,421 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:14,230 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:14,230 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:17,234 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:17,978 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:17,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:17,981 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:17,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:17,983 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:18,236 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:19,237 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:19,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:44:19,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:44:19,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:44:20,239 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:22,545 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:22,546 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:22,546 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:22,548 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:23,243 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:23,244 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:23,457 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:24,245 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:27,120 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:27,121 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:27,121 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:27,123 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:27,249 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:27,250 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:28,251 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:28,860 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:44:28,860 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:31,255 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:31,688 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:31,689 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:31,690 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:31,692 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:32,257 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:32,257 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:34,432 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:34,921 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:44:34,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:44:34,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:44:35,261 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:36,258 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:36,260 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:36,260 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:36,262 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:36,263 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:37,264 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:38,266 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:39,916 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:40,829 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:40,830 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:40,830 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:40,832 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:41,270 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:41,271 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:42,272 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:45,276 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:45,408 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:45,409 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:45,410 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:45,410 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:45,412 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:46,278 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:46,278 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:49,282 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:49,922 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:44:49,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:44:49,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:44:50,012 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:50,013 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:50,013 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:50,015 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:50,284 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:50,896 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:51,286 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:52,287 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:54,548 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:54,549 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:54,549 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:54,552 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:55,292 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:55,292 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:56,293 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:44:56,373 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:44:58,861 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:44:59,112 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:44:59,113 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:44:59,113 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:44:59,115 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:44:59,297 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:44:59,298 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:00,299 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:01,851 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:03,302 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:03,686 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:03,687 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:03,688 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:03,689 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:04,306 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:04,306 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:04,922 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:45:04,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:45:04,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:45:07,310 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:07,339 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:08,253 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:08,253 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:08,254 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:08,256 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:08,312 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:09,313 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:10,315 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:12,828 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:12,829 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:12,830 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:12,830 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:12,832 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:13,320 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:13,320 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:14,321 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:17,325 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:17,396 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:17,397 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:17,398 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:17,400 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:18,310 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:18,327 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:18,327 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:19,922 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:45:19,922 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:45:19,958 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:45:21,331 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:21,971 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:21,972 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:21,972 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:21,974 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:22,333 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:23,334 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:23,801 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:24,336 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:26,837 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:26,838 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:26,839 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:26,840 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:27,341 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:27,341 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:28,342 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:28,901 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:28,954 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:45:32,347 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:34,027 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:35,049 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:45:35,049 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:45:35,050 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:45:36,353 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:39,357 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:39,935 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:45,310 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:45,365 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:46,222 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:46,224 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:46,224 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:46,226 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:46,368 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:47,369 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:48,371 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:50,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:45:50,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:45:50,049 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:45:50,826 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:51,374 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:51,683 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:51,684 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:51,684 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:51,686 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:52,377 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:52,377 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:55,347 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:55,348 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:55,349 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:55,351 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:45:55,381 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:45:55,382 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:56,383 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:57,169 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:45:58,865 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:45:59,386 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:45:59,911 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:45:59,912 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:45:59,913 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:45:59,914 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:00,388 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:01,390 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:02,391 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:02,657 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:04,486 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:04,487 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:04,488 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:04,489 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:05,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:46:05,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:46:05,049 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:46:05,396 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:05,396 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:06,397 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:08,147 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:09,058 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:09,059 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:09,059 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:09,061 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:09,402 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:09,402 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:10,403 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:13,410 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:13,629 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:13,631 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:13,631 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:13,632 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:13,636 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:14,412 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:14,413 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:17,416 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:18,205 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:18,206 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:18,206 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:18,208 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:18,419 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:19,118 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:19,420 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:20,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:46:20,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:46:20,058 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:46:20,423 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:22,775 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:22,777 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:22,777 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:22,779 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:23,428 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:23,428 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:24,429 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:24,605 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:27,348 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:27,349 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:27,349 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:27,351 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:27,434 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:27,434 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:28,436 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:28,867 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:46:30,089 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:31,439 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:31,917 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:31,919 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:31,919 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:31,921 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:32,442 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:33,443 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:34,444 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:35,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:46:35,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:46:35,054 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:46:35,097 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:36,483 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:36,484 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:36,485 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:36,486 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:37,449 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:37,449 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:38,450 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:40,139 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:41,052 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:41,053 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:41,053 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:41,055 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:41,455 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:41,456 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:42,457 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:45,461 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:45,624 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:45,625 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:45,626 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:45,626 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:45,628 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:46,463 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:46,463 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:49,467 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:50,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:46:50,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:46:50,054 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:46:50,238 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:51,012 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:51,012 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:51,013 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:51,014 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:51,470 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:51,471 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:52,472 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:54,767 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:54,768 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:54,768 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:54,770 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:55,477 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:55,477 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:56,478 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:46:56,591 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:46:58,869 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:46:59,332 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:46:59,333 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:46:59,333 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:46:59,335 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:46:59,483 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:46:59,483 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:00,484 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:02,072 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:03,488 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:03,905 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:03,906 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:03,907 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:03,909 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:04,491 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:05,021 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:47:05,021 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:47:05,054 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:47:05,492 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:06,493 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:07,565 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:08,481 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:08,482 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:08,483 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:08,485 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:08,497 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:09,498 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:10,499 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:13,050 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:13,051 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:13,052 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:13,052 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:13,054 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:13,504 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:13,504 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:14,505 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:18,373 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:18,511 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:20,830 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:47:20,830 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:47:20,831 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:47:22,518 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:23,478 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:25,521 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:27,524 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:28,710 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:28,871 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:47:31,531 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:32,347 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:32,349 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:32,349 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:32,351 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:32,533 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:33,535 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:34,171 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:34,537 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:35,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:47:35,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:47:35,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:47:36,906 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:36,907 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:36,908 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:36,912 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:37,547 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:37,547 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:38,549 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:39,636 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:41,486 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:41,487 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:41,488 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:41,498 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:41,560 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:41,560 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:42,562 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:45,135 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:45,570 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:46,052 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:46,053 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:46,053 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:46,063 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:46,575 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:47,578 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:48,580 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:50,626 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:50,632 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:50,633 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:50,634 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:50,639 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:50,787 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:47:50,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:47:50,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:47:51,596 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:51,596 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:52,598 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:55,212 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:55,214 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:55,214 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:55,224 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:47:55,610 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:47:55,610 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:56,124 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:47:56,613 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:58,879 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:47:59,620 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:47:59,797 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:47:59,799 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:47:59,799 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:47:59,801 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:00,625 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:01,620 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:01,627 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:02,629 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:04,366 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:04,367 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:04,368 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:04,369 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:04,634 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:05,635 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:05,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:48:05,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:48:05,830 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:48:06,637 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:07,105 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:08,934 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:08,935 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:08,936 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:08,938 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:09,646 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:09,647 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:10,648 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:12,590 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:13,507 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:13,508 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:13,509 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:13,511 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:13,653 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:13,653 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:14,654 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:17,658 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:18,084 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:18,085 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:18,086 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:18,087 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:18,089 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:18,660 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:19,662 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:20,663 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:20,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:48:20,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:48:20,830 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:48:22,694 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:22,696 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:22,696 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:22,698 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:23,606 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:23,668 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:23,668 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:24,670 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:27,273 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:27,274 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:27,275 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:27,281 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:27,678 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:27,678 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:28,680 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:28,876 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:48:28,876 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:31,685 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:33,112 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:33,114 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:33,114 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:33,122 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:33,691 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:33,691 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:34,072 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:34,693 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:35,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:48:35,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:48:35,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:48:37,698 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:37,900 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:37,901 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:37,902 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:37,904 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:38,700 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:39,702 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:39,818 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:40,703 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:42,697 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:42,698 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:42,699 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:42,700 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:42,707 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:43,708 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:44,709 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:45,550 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:47,442 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:47,443 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:47,443 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:47,445 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:47,714 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:47,714 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:48,715 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:50,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:50,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:48:50,788 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:48:50,788 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:48:51,719 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:52,163 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:52,164 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:52,164 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:52,166 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:52,722 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:53,723 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:54,724 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:55,950 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:48:56,895 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:48:56,897 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:48:56,897 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:48:56,899 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:48:57,730 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:48:57,730 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:58,731 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:48:58,877 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:49:01,605 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:01,606 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:01,608 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:01,608 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:01,610 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:01,736 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:01,736 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:02,737 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:05,741 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:49:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:49:05,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:49:07,214 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:07,744 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:08,746 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:12,336 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:12,751 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:15,755 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:17,828 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:19,760 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:20,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:49:20,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:49:20,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:49:21,618 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:21,619 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:21,620 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:21,621 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:21,763 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:21,763 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:22,765 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:23,450 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:25,769 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:26,250 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:26,251 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:26,251 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:26,253 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:26,771 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:27,772 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:28,774 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:28,879 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:49:28,880 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:30,912 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:30,913 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:30,914 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:30,915 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:31,779 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:31,779 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:32,780 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:34,646 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:35,579 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:35,580 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:35,581 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:35,583 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:35,785 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:35,785 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:35,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:49:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:49:35,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:49:36,786 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:39,790 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:40,251 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:40,252 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:40,253 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:40,253 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:40,256 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:40,792 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:41,794 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:42,795 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:44,910 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:44,911 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:44,911 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:44,913 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:45,800 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:45,800 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:45,841 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:46,801 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:49,554 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:49,555 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:49,555 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:49,557 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:49,806 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:49,806 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:50,807 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:50,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:49:50,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:49:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:50,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:49:53,811 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:54,212 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:54,213 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:54,214 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:54,216 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:54,813 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:55,815 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:56,082 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:49:56,816 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:49:58,880 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:49:58,882 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:49:58,883 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:49:58,883 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:49:58,884 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:49:59,821 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:49:59,821 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:00,822 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:01,678 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:02,825 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:03,542 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:03,543 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:03,544 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:03,546 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:03,827 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:04,828 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:05,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:50:05,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:50:05,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:50:06,831 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:07,275 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:08,210 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:08,211 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:08,212 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:08,213 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:08,835 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:08,835 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:10,837 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:12,840 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:12,871 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:12,872 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:12,873 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:12,873 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:12,875 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:13,842 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:14,843 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:16,846 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:17,540 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:17,541 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:17,541 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:17,543 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:17,848 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:18,469 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:18,849 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:20,852 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:20,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:50:20,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:50:20,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:50:22,179 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:22,189 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:22,189 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:22,191 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:22,856 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:22,856 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:24,047 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:24,858 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:26,823 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:26,824 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:26,825 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:26,828 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:26,862 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:26,862 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:28,864 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:28,941 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:50:29,608 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:30,867 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:31,471 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:31,473 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:31,473 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:31,475 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:31,870 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:32,870 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:34,718 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:34,873 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:35,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:50:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:50:35,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:50:36,618 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:36,619 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:36,620 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:36,621 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:36,876 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:36,877 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:38,879 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:40,303 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:40,881 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:41,229 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:41,230 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:41,231 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:41,233 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:41,884 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:42,885 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:44,888 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:45,866 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:45,867 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:45,868 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:45,869 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:45,871 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:45,890 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:46,891 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:48,893 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:50,521 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:50:50,522 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:50:50,522 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:50:50,525 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:50:50,897 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:50:50,897 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:50:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:50:50,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:50:52,899 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:56,673 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:50:56,905 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:50:58,942 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:51:00,910 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:02,660 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:02,913 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:05,979 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:51:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:51:05,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:51:06,918 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:08,062 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:08,920 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:09,907 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:09,993 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:09,993 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:09,995 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:10,924 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:10,924 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:12,926 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:13,621 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:14,549 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:14,550 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:14,551 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:14,553 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:14,930 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:14,930 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:16,932 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:18,935 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:19,179 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:19,180 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:19,182 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:19,182 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:19,184 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:19,937 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:20,939 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:20,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:51:20,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:51:20,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:51:22,941 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:23,824 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:23,825 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:23,826 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:23,828 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:23,944 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:24,747 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:24,945 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:26,947 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:28,465 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:28,466 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:28,466 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:28,468 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:28,944 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:51:28,951 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:28,951 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:30,320 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:30,954 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:32,956 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:33,105 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:33,106 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:33,106 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:33,108 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:33,959 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:34,960 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:35,884 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:51:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:51:35,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:51:36,962 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:37,739 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:37,740 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:37,740 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:37,742 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:37,965 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:38,966 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:40,968 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:41,448 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:42,378 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:42,380 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:42,380 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:42,382 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:42,972 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:42,972 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:44,975 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:46,977 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:47,031 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:47,032 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:47,034 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:47,034 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:47,036 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:47,979 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:48,981 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:51:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:51:50,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:51:50,983 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:51,942 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:51,943 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:51,944 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:51,946 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:51,985 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:52,870 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:52,987 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:54,989 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:56,605 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:51:56,607 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:51:56,607 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:51:56,609 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:51:56,993 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:51:56,993 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:51:58,470 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:51:58,946 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:51:58,995 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:00,998 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:01,254 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:01,255 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:01,256 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:01,258 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:02,000 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:03,002 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:04,035 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:05,004 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:05,898 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:05,899 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:05,899 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:05,901 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:52:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:52:05,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:52:06,006 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:07,008 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:09,010 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:09,616 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:10,548 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:10,550 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:10,550 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:10,552 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:11,014 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:11,014 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:13,016 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:15,019 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:15,188 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:15,189 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:15,190 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:15,191 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:15,193 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:16,022 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:17,023 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:19,025 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:19,830 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:19,831 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:19,832 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:19,834 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:20,028 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:20,754 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:20,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:52:20,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:52:20,980 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:52:21,029 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:23,032 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:24,474 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:24,475 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:24,475 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:24,478 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:25,035 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:25,035 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:26,325 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:27,038 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:28,949 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:52:29,040 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:29,096 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:29,098 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:29,098 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:29,100 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:30,043 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:31,044 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:31,873 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:33,046 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:33,730 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:33,731 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:33,732 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:33,734 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:34,049 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:35,050 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:52:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:52:35,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:52:37,053 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:37,437 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:38,361 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:38,362 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:38,363 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:38,364 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:39,056 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:39,056 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:41,058 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:42,455 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:45,064 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:48,054 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:49,070 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:52:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:52:50,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:52:51,072 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:53,121 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:55,077 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:57,080 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:52:58,223 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:52:58,224 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:52:58,225 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:52:58,226 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:52:58,228 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:52:58,951 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:52:59,084 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:52:59,084 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:01,086 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:02,869 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:02,871 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:02,871 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:02,873 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:03,090 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:03,090 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:03,803 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:05,092 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:53:05,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:53:05,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:53:07,095 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:07,510 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:07,512 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:07,512 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:07,514 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:08,097 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:09,099 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:09,377 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:11,101 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:12,163 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:12,165 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:12,165 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:12,167 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:13,105 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:13,105 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:14,966 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:15,107 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:16,826 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:16,827 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:16,828 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:16,830 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:17,112 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:17,112 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:19,115 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:20,528 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:20,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:53:20,981 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:53:20,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:53:21,117 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:21,490 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:21,492 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:21,492 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:21,494 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:22,120 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:23,121 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:25,124 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:26,093 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:26,094 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:26,095 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:26,096 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:26,097 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:26,126 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:27,127 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:28,952 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:53:29,130 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:30,741 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:30,743 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:30,743 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:30,746 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:31,133 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:31,134 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:31,670 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:33,136 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:35,139 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:35,392 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:35,393 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:35,394 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:35,396 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:35,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:53:35,981 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:53:35,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:53:36,141 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:37,142 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:37,244 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:39,145 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:40,027 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:40,028 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:40,029 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:40,031 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:40,147 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:41,148 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:42,807 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:43,151 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:44,661 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:44,663 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:44,663 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:44,665 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:45,155 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:45,155 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:47,157 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:48,367 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:49,160 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:49,288 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:49,290 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:49,290 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:49,292 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:50,162 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:50,980 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:53:50,981 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:53:50,981 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:53:51,163 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:53,166 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:53,931 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:53,931 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:53,933 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:53,933 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:53,935 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:54,168 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:55,169 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:57,172 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:53:58,550 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:53:58,552 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:53:58,552 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:53:58,554 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:53:58,955 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:53:58,956 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:53:59,176 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:53:59,176 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:01,178 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:03,181 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:03,184 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:03,186 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:03,186 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:03,188 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:04,111 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:04,183 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:05,185 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:06,000 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:54:06,002 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:54:06,002 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:54:07,187 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:07,821 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:07,822 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:07,822 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:07,824 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:08,189 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:09,191 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:09,669 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:11,193 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:12,451 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:12,452 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:12,453 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:12,454 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:13,197 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:13,197 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:15,199 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:15,220 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:17,072 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:17,074 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:17,074 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:17,076 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:17,203 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:17,203 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:19,205 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:20,771 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:20,991 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:54:21,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:54:21,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:54:21,208 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:21,970 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:21,971 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:21,972 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:21,973 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:22,210 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:23,212 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:25,214 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:26,601 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:26,602 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:26,603 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:26,604 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:26,606 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:27,218 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:27,218 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:28,957 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:54:29,220 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:31,791 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:33,226 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:35,991 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:54:36,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:54:36,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:54:37,143 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:37,231 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:39,233 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:42,405 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:43,238 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:45,241 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:46,097 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:46,099 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:46,099 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:46,101 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:46,243 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:47,245 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:47,939 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:49,247 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:50,708 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:50,709 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:50,710 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:50,712 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:50,992 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:54:51,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:54:51,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:54:51,251 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:51,251 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:53,254 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:53,475 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:55,256 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:55,324 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:55,325 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:55,326 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:55,328 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:54:56,259 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:54:57,260 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:58,974 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:54:58,975 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:54:59,263 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:54:59,946 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:54:59,947 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:54:59,947 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:54:59,949 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:00,265 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:01,266 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:03,269 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:04,577 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:04,578 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:04,580 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:04,580 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:04,582 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:05,273 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:05,273 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:05,991 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:55:06,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:55:06,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:55:07,275 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:09,205 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:09,206 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:09,206 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:09,208 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:09,279 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:09,279 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:10,127 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:11,282 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:13,284 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:13,834 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:13,836 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:13,836 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:13,838 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:14,287 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:15,288 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:15,685 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:17,291 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:18,463 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:18,464 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:18,464 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:18,466 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:19,294 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:19,295 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:20,992 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:55:21,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:21,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:55:21,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:55:21,297 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:23,104 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:23,105 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:23,106 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:23,108 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:23,301 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:23,301 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:24,302 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:26,821 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:27,306 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:27,752 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:27,753 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:27,754 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:27,755 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:28,309 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:28,310 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:28,961 DEBUG SenderThread:237521 [sender.py:send():382] send: stats -2024-02-01 17:55:31,313 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:32,374 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:32,375 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:32,376 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:32,376 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:32,378 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:33,316 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:33,317 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:34,318 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:35,992 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-01 17:55:36,001 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status -2024-02-01 17:55:36,001 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: stop_status -2024-02-01 17:55:37,011 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:37,012 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:37,012 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:37,014 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-02-01 17:55:37,323 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json -2024-02-01 17:55:37,323 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:37,940 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report -2024-02-01 17:55:38,324 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:41,328 INFO Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log -2024-02-01 17:55:41,644 DEBUG HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history -2024-02-01 17:55:41,645 DEBUG SenderThread:237521 [sender.py:send():382] send: history -2024-02-01 17:55:41,645 DEBUG SenderThread:237521 [sender.py:send_request():409] send_request: summary_record -2024-02-01 17:55:41,647 INFO SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 17:58:50,082 INFO StreamThr :239784 [internal.py:wandb_internal():86] W&B internal server running at pid: 239784, started at: 2024-02-01 17:58:50.080918 +2024-02-01 17:58:50,083 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status +2024-02-01 17:58:50,087 INFO WriterThread:239784 [datastore.py:open_for_write():85] open: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/run-i93q0p12.wandb +2024-02-01 17:58:50,088 DEBUG SenderThread:239784 [sender.py:send():382] send: header +2024-02-01 17:58:50,109 DEBUG SenderThread:239784 [sender.py:send():382] send: run +2024-02-01 17:58:50,401 INFO SenderThread:239784 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files +2024-02-01 17:58:50,401 INFO SenderThread:239784 [sender.py:_start_run_threads():1136] run started: i93q0p12 with start time 1706810330.083967 +2024-02-01 17:58:50,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: check_version +2024-02-01 17:58:50,409 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: check_version +2024-02-01 17:58:50,463 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: run_start +2024-02-01 17:58:50,558 DEBUG HandlerThread:239784 [system_info.py:__init__():32] System info init +2024-02-01 17:58:50,558 DEBUG HandlerThread:239784 [system_info.py:__init__():47] System info init done +2024-02-01 17:58:50,558 INFO HandlerThread:239784 [system_monitor.py:start():194] Starting system monitor +2024-02-01 17:58:50,558 INFO SystemMonitor:239784 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-01 17:58:50,559 INFO HandlerThread:239784 [system_monitor.py:probe():214] Collecting system info +2024-02-01 17:58:50,559 INFO SystemMonitor:239784 [interfaces.py:start():190] Started cpu monitoring +2024-02-01 17:58:50,559 INFO SystemMonitor:239784 [interfaces.py:start():190] Started disk monitoring +2024-02-01 17:58:50,560 INFO SystemMonitor:239784 [interfaces.py:start():190] Started gpu monitoring +2024-02-01 17:58:50,560 INFO SystemMonitor:239784 [interfaces.py:start():190] Started memory monitoring +2024-02-01 17:58:50,561 INFO SystemMonitor:239784 [interfaces.py:start():190] Started network monitoring +2024-02-01 17:58:50,615 DEBUG HandlerThread:239784 [system_info.py:probe():196] Probing system +2024-02-01 17:58:50,618 DEBUG HandlerThread:239784 [system_info.py:_probe_git():181] Probing git +2024-02-01 17:58:50,642 DEBUG HandlerThread:239784 [system_info.py:_probe_git():189] Probing git done +2024-02-01 17:58:50,642 DEBUG HandlerThread:239784 [system_info.py:probe():244] Probing system done +2024-02-01 17:58:50,642 DEBUG HandlerThread:239784 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.5', 'heartbeatAt': '2024-02-01T17:58:50.615903', 'startedAt': '2024-02-01T17:58:50.064651', 'docker': None, 'cuda': None, 'args': ('config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co./sanchit-gandhi/distil-zephyr-1.5b-ssft', 'commit': '79a4ae874a71e67016ded927e7d23351e5c7dab8'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft', 'host': 'ip-26-0-165-24', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2728.6971041666666, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3589.018, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3587.597, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3596.958, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3597.83, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3595.048, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3591.991, 'min': 0.0, 'max': 0.0}, {'current': 3598.381, 'min': 0.0, 'max': 0.0}, {'current': 3598.099, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 57.44965744018555}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855346679688}} +2024-02-01 17:58:50,642 INFO HandlerThread:239784 [system_monitor.py:probe():224] Finished collecting system info +2024-02-01 17:58:50,642 INFO HandlerThread:239784 [system_monitor.py:probe():227] Publishing system info +2024-02-01 17:58:50,642 DEBUG HandlerThread:239784 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2024-02-01 17:58:50,644 DEBUG HandlerThread:239784 [system_info.py:_save_pip():68] Saving pip packages done +2024-02-01 17:58:50,644 DEBUG HandlerThread:239784 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2024-02-01 17:58:51,404 INFO Thread-12 :239784 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/requirements.txt +2024-02-01 17:58:51,404 INFO Thread-12 :239784 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/conda-environment.yaml +2024-02-01 17:58:55,310 DEBUG HandlerThread:239784 [system_info.py:_save_conda():87] Saving conda packages done +2024-02-01 17:58:55,312 INFO HandlerThread:239784 [system_monitor.py:probe():229] Finished publishing system info +2024-02-01 17:58:55,339 DEBUG SenderThread:239784 [sender.py:send():382] send: files +2024-02-01 17:58:55,339 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2024-02-01 17:58:55,339 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:58:55,347 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 17:58:55,347 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 17:58:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 17:58:55,408 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/conda-environment.yaml +2024-02-01 17:58:55,409 INFO Thread-12 :239784 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-metadata.json +2024-02-01 17:58:55,462 DEBUG SenderThread:239784 [sender.py:send():382] send: telemetry +2024-02-01 17:58:55,463 DEBUG SenderThread:239784 [sender.py:send():382] send: config +2024-02-01 17:58:55,463 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 17:58:55,463 DEBUG SenderThread:239784 [sender.py:send():382] send: telemetry +2024-02-01 17:58:55,463 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 17:58:55,464 WARNING SenderThread:239784 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2024-02-01 17:58:55,464 DEBUG SenderThread:239784 [sender.py:send():382] send: telemetry +2024-02-01 17:58:55,631 INFO wandb-upload_0:239784 [upload_job.py:push():131] Uploaded file /tmp/tmpr1qr0ps0wandb/r53qkhjh-wandb-metadata.json +2024-02-01 17:58:56,411 INFO Thread-12 :239784 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:58:58,413 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:00,471 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:05,472 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:06,418 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 17:59:06,420 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 17:59:06,420 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 17:59:06,421 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 17:59:06,421 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 17:59:06,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 17:59:06,424 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 17:59:06,424 INFO Thread-12 :239784 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 17:59:08,427 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:10,347 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 17:59:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 17:59:10,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 17:59:10,493 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:12,432 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:15,576 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:16,437 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:18,439 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:20,788 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:21,444 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/config.yaml +2024-02-01 17:59:22,446 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:25,347 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 17:59:25,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 17:59:25,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 17:59:26,451 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:26,471 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:30,456 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:31,603 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:34,461 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:36,464 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:37,209 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:38,807 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 17:59:38,808 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 17:59:38,808 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 17:59:38,812 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 17:59:39,469 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 17:59:40,347 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 17:59:40,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 17:59:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 17:59:40,470 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:42,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:44,475 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:48,018 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:48,480 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:50,561 DEBUG SystemMonitor:239784 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-01 17:59:50,564 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 17:59:52,485 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:53,221 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 17:59:54,488 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:55,347 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 17:59:55,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 17:59:55,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 17:59:57,492 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 17:59:58,833 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:01,497 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:04,040 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:05,502 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:09,507 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:09,647 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:00:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:00:10,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:00:11,509 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:14,847 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:14,849 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:00:14,850 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:00:14,851 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:00:14,853 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:00:15,516 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:15,516 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:00:16,517 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:19,521 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:20,459 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:20,566 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:00:23,526 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:25,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:00:25,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:00:25,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:00:25,478 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:27,531 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:29,534 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:31,279 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:33,539 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:36,505 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:37,544 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:00:40,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:00:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:00:41,549 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:42,498 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:45,554 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:47,556 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:48,306 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:50,568 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:00:50,912 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:00:50,913 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:00:50,914 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:00:50,915 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:00:51,562 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:51,563 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:00:52,564 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:53,925 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:55,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:00:55,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:00:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:00:55,567 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:00:59,109 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:00:59,572 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:03,578 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:04,714 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:05,580 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:09,585 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:09,913 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:01:10,348 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:01:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:01:13,590 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:15,510 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:17,595 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:19,526 WARNING FileStreamThread:239784 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/sanchit-gandhi/huggingface/i93q0p12/file_stream. func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/sanchit-gandhi/huggingface/i93q0p12/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 7.6246, "train/learning_rate": 4.3795620437956196e-05, "train/epoch": 0.07, "train/global_step": 20, "_timestamp": 1706810414.8489923, "_runtime": 84.76502537727356, "_step": 2}']}, 'wandb-history.jsonl': {'offset': 2, 'content': ['{"train/loss": 7.6246, "train/learning_rate": 4.3795620437956196e-05, "train/epoch": 0.07, "train/global_step": 20, "_timestamp": 1706810414.8489923, "_runtime": 84.76502537727356, "_step": 2}']}, 'output.log': {'offset': 5, 'content': ["2024-02-01T18:00:16.385360 {'loss': 7.6246, 'learning_rate': 4.3795620437956196e-05, 'epoch': 0.07}\n"]}}, 'dropped': 0}} +2024-02-01 18:01:20,570 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:01:20,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:21,600 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:23,603 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:25,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:01:25,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:01:25,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:01:26,448 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:26,942 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:01:26,943 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:01:26,944 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:01:26,945 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:01:27,609 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:27,609 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:01:28,611 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:31,497 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:31,615 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:35,620 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:37,093 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:39,625 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:01:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:01:40,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:01:41,628 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:42,310 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:45,634 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:47,915 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:49,639 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:50,572 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:01:53,132 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:53,644 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:55,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:01:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:01:55,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:01:57,650 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:01:58,727 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:01:59,652 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:02,945 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:02:02,947 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:02:02,947 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:02:02,949 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:02:03,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:03,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:02:03,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:04,660 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:07,664 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:09,549 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:10,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:02:10,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:02:10,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:02:11,669 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:14,750 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:15,675 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:17,677 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:20,345 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:20,574 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:02:21,683 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:25,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:02:25,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:02:25,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:02:25,418 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:25,688 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:29,693 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:31,181 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:33,698 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:35,701 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:36,400 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:39,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:02:39,024 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:02:39,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:02:39,026 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:02:39,707 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:39,707 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:02:40,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:02:40,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:02:40,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:02:40,708 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:41,400 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:43,712 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:47,226 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:47,718 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:50,576 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:02:51,723 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:52,577 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:02:53,726 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:02:55,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:02:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:02:57,731 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:02:58,047 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:01,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:03,651 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:05,742 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:08,868 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:09,748 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:10,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:03:10,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:03:10,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:03:11,750 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:14,469 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:15,092 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:03:15,093 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:03:15,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:03:15,096 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:03:15,757 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:15,757 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:03:16,758 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:19,706 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:19,762 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:20,577 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:03:23,767 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:25,309 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:25,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:03:25,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:03:25,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:03:27,773 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:30,545 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:31,778 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:33,781 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:36,152 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:37,786 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:40,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:03:40,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:03:40,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:03:41,404 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:41,791 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:45,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:46,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:49,802 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:50,579 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:03:51,183 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:03:51,184 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:03:51,185 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:03:51,187 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:03:51,806 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:51,806 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:03:52,187 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:52,807 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:03:55,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:03:55,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:03:55,811 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:03:57,472 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:03:59,817 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:03,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:03,822 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:07,828 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:08,620 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:09,830 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:10,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:04:10,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:04:10,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:04:13,836 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:13,836 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:17,841 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:19,433 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:20,581 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:04:21,847 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:24,633 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:25,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:04:25,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:04:25,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:04:25,852 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:27,274 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:04:27,275 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:04:27,275 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:04:27,277 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:04:27,855 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:27,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:04:28,857 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:30,278 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:31,861 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:35,451 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:35,866 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:39,871 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:40,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:04:40,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:04:40,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:04:40,479 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:43,877 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:45,879 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:46,271 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:49,884 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:50,583 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:04:51,584 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:53,890 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:04:55,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:04:55,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:04:55,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:04:57,096 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:04:57,895 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:01,900 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:02,711 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:03,319 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:05:03,320 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:05:03,320 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:05:03,322 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:05:03,904 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:03,904 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:05:04,905 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:07,909 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:07,930 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:10,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:05:10,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:05:10,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:05:11,915 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:13,541 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:15,920 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:18,751 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:19,925 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:20,595 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:05:21,928 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:24,364 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:25,349 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:05:25,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:05:25,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:05:25,933 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:29,563 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:29,937 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:33,942 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:35,165 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:37,947 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:39,380 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:05:39,381 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:05:39,381 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:05:39,383 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:05:39,951 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:39,951 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:05:40,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:05:40,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:05:40,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:05:40,443 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:40,952 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:43,956 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:45,987 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:47,961 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:50,587 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:05:51,587 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:51,966 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:55,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:05:55,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:05:55,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:05:55,970 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:05:57,430 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:05:57,973 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:01,978 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:03,038 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:05,983 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:08,254 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:09,988 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:10,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:06:10,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:06:10,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:06:13,860 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:13,993 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:15,460 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:06:15,461 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:06:15,462 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:06:15,464 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:06:15,996 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:15,997 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:06:16,998 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:19,075 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:20,001 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:20,589 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:06:24,006 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:24,688 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:25,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:06:25,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:06:25,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:06:28,012 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:29,911 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:32,017 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:34,019 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:35,529 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:38,025 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:40,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:06:40,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:06:40,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:06:40,741 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:42,029 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:46,035 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:46,345 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:50,039 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:50,590 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:06:51,549 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:06:51,550 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:06:51,551 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:06:51,552 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:06:51,554 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:06:52,043 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:52,043 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:06:53,044 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:55,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:06:55,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:06:55,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:06:56,048 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:06:57,499 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:00,053 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:03,326 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:04,058 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:08,063 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:08,919 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:10,065 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:10,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:07:10,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:07:10,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:07:14,071 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:14,115 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:18,076 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:19,718 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:20,602 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:07:22,081 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:24,919 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:25,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:07:25,350 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:07:25,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:07:26,086 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:27,566 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:07:27,567 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:07:27,567 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:07:27,569 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:07:28,089 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:28,090 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:07:29,091 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:30,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:32,095 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:35,723 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:36,101 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:40,107 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:40,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:07:40,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:07:40,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:07:41,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:44,119 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:46,123 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:47,458 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:50,133 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:50,594 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:07:52,595 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:07:54,144 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:55,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:07:55,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:07:55,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:07:58,154 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:07:58,183 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:02,165 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:03,329 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:03,330 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:08:03,331 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:08:03,331 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:08:03,346 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:08:04,175 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:04,175 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:08:05,177 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:08,184 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:08,914 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:10,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:08:10,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:08:10,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:08:12,193 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:14,072 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:16,203 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:19,644 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:20,210 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:20,596 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:08:22,213 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:24,790 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:25,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:08:25,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:08:25,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:08:26,218 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:30,223 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:30,353 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:34,228 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:35,552 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:36,230 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:39,143 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:08:39,144 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:08:39,144 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:08:39,146 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:08:39,235 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:08:40,236 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:40,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:08:40,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:08:40,353 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:08:41,238 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:41,422 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:44,242 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:47,289 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:48,247 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:50,600 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:08:52,260 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:52,602 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:54,264 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:08:55,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:08:55,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:08:55,355 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:08:58,269 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:08:58,272 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:02,277 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:03,969 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:06,282 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:09,260 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:10,287 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:10,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:09:10,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:09:10,355 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:09:14,292 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:14,879 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:15,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:09:15,520 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:09:15,520 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:09:15,524 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:09:16,295 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:16,296 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:09:17,297 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:20,151 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:20,301 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:20,601 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:09:24,306 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:25,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:09:25,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:09:25,353 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:09:25,438 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:28,311 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:31,008 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:32,315 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:36,320 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:36,632 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:38,323 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:40,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:09:40,351 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:09:40,353 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:09:41,854 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:42,328 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:46,333 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:47,458 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:50,338 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:50,602 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:09:51,687 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:09:51,689 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:09:51,689 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:09:51,691 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:09:52,341 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:09:52,692 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:09:53,343 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:54,344 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:55,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:09:55,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:09:55,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:09:56,347 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:09:58,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:00,352 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:04,357 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:04,511 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:08,362 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:09,721 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:10,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:10:10,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:10:10,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:10:12,367 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:14,369 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:15,328 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:18,374 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:20,510 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:20,604 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:10:22,380 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:25,383 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:10:25,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:10:25,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:10:26,387 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:26,498 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:27,754 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:10:27,755 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:10:27,755 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:10:27,757 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:10:28,390 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:10:29,391 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:30,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:32,302 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:32,395 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:36,400 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:37,916 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:40,383 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:10:40,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:10:40,405 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:40,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:10:43,116 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:44,411 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:48,416 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:48,718 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:50,418 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:50,606 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:10:53,902 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:10:54,423 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:55,383 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:10:55,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:10:55,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:10:58,428 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:10:59,501 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:02,433 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:03,689 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:11:03,691 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:11:03,691 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:11:03,693 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:11:04,437 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:11:04,693 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:05,438 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:06,440 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:08,442 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:10,295 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:10,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:11:10,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:11:10,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:11:12,447 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:15,474 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:16,452 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:20,457 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:20,608 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:11:20,609 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:24,462 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:25,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:11:25,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:11:25,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:11:26,465 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:26,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:30,470 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:31,839 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:34,475 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:37,033 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:38,480 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:39,627 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:11:39,628 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:11:39,628 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:11:39,630 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:11:40,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:11:40,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:11:40,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:11:40,483 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:40,483 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:11:42,527 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:44,488 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:47,808 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:48,493 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:50,619 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:11:52,498 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:53,620 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:11:55,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:11:55,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:11:55,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:11:56,503 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:58,505 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:11:59,589 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:02,510 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:04,771 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:06,515 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:10,369 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:10,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:12:10,384 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:12:10,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:12:10,520 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:14,525 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:15,549 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:15,550 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:12:15,551 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:12:15,552 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:12:15,554 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:12:16,530 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:16,530 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:12:20,534 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:20,612 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:12:20,613 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:24,539 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:25,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:12:25,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:12:25,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:12:26,333 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:28,544 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:31,919 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:32,549 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:34,552 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:37,107 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:38,557 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:40,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:12:40,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:12:40,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:12:42,562 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:42,706 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:46,567 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:47,896 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:50,571 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:50,614 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:12:51,492 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:12:51,493 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:12:51,494 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:12:51,496 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:12:51,574 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:12:52,575 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:53,496 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:12:55,384 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:12:55,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:12:55,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:12:56,580 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:12:58,690 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:00,585 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:04,277 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:04,590 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:08,595 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:09,448 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:10,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:13:10,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:13:10,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:13:10,598 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:14,603 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:15,033 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:18,608 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:20,216 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:20,616 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:13:22,613 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:25,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:13:25,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:13:25,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:13:25,501 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:26,618 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:27,430 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:13:27,431 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:13:27,432 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:13:27,433 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:13:27,620 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:13:28,621 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:30,976 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:32,626 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:36,561 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:36,631 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:40,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:13:40,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:13:40,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:13:40,636 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:41,750 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:44,641 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:46,644 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:47,330 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:50,628 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:13:50,649 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:52,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:54,654 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:13:55,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:13:55,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:13:55,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:13:58,110 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:13:58,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:02,664 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:03,307 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:03,308 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:14:03,309 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:14:03,309 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:14:03,311 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:14:03,666 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:14:04,667 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:08,672 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:08,892 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:10,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:14:10,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:14:10,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:14:12,678 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:14,071 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:16,683 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:18,685 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:19,651 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:20,619 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:14:22,690 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:24,814 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:25,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:14:25,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:14:25,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:14:26,695 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:30,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:30,700 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:34,705 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:35,707 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:38,711 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:39,365 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:14:39,366 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:14:39,366 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:14:39,368 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:14:39,713 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:14:40,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:14:40,385 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:14:40,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:14:40,715 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:41,513 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:44,720 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:46,636 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:48,724 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:50,621 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:14:51,729 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:52,622 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:14:55,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:14:55,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:14:55,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:14:55,734 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:57,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:14:58,544 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:01,741 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:04,165 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:05,746 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:09,426 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:09,751 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:10,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:15:10,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:15:10,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:15:13,756 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:15,048 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:15,680 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:15:15,681 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:15:15,681 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:15:15,683 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:15:15,760 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:15,760 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:15:16,761 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:19,765 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:20,298 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:20,623 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:15:23,770 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:25,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:15:25,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:15:25,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:15:25,508 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:27,775 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:29,778 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:30,539 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:31,781 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:33,783 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:35,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:35,786 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:37,789 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:39,791 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:40,385 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:15:40,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:15:40,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:15:40,590 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:41,794 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:43,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:45,601 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:45,799 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:47,802 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:49,804 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:50,626 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:15:50,627 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:51,807 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:53,810 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:55,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:15:55,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:15:55,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:15:55,640 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:15:55,812 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:57,215 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:15:57,219 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 18:15:57,220 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 18:15:57,220 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 18:15:57,221 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 18:15:57,221 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:15:57,221 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:15:57,223 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:15:57,816 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:57,816 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:15:58,817 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:15:59,819 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:01,327 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:01,821 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:02,823 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:06,828 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:06,938 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:09,832 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:10,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:16:10,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:16:10,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:16:12,143 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:12,837 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/config.yaml +2024-02-01 18:16:13,838 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:15,841 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:17,724 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:19,846 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:20,627 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:16:22,891 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:23,851 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:25,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:16:25,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:16:25,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:16:27,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:28,490 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:31,861 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:33,645 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:33,646 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:16:33,647 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:16:33,647 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:16:33,650 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:16:33,865 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:33,865 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:16:34,866 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:37,870 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:39,229 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:40,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:16:40,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:16:40,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:16:41,875 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:44,390 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:45,881 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:49,886 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:49,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:50,629 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:16:51,888 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:55,146 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:16:55,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:16:55,386 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:16:55,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:16:55,893 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:16:59,899 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:00,735 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:03,904 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:05,905 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:07,909 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:09,494 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:17:09,495 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:17:09,495 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:17:09,497 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:17:09,913 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:09,913 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:17:10,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:17:10,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:17:10,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:17:10,914 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:11,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:13,918 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log