diff --git "a/wandb/run-20240922_051551-1efs7bbc/logs/debug-internal.log" "b/wandb/run-20240922_051551-1efs7bbc/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240922_051551-1efs7bbc/logs/debug-internal.log" @@ -0,0 +1,1650 @@ +2024-09-22 05:15:51,146 INFO StreamThr :1229619 [internal.py:wandb_internal():85] W&B internal server running at pid: 1229619, started at: 2024-09-22 05:15:51.145041 +2024-09-22 05:15:51,148 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status +2024-09-22 05:15:51,150 INFO WriterThread:1229619 [datastore.py:open_for_write():87] open: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/run-1efs7bbc.wandb +2024-09-22 05:15:51,151 DEBUG SenderThread:1229619 [sender.py:send():379] send: header +2024-09-22 05:15:51,171 DEBUG SenderThread:1229619 [sender.py:send():379] send: run +2024-09-22 05:15:51,596 INFO SenderThread:1229619 [dir_watcher.py:__init__():211] watching files in: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files +2024-09-22 05:15:51,597 INFO SenderThread:1229619 [sender.py:_start_run_threads():1188] run started: 1efs7bbc with start time 1726953351.147649 +2024-09-22 05:15:51,600 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: check_version +2024-09-22 05:15:51,600 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: check_version +2024-09-22 05:15:51,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: run_start +2024-09-22 05:15:51,850 DEBUG HandlerThread:1229619 [system_info.py:__init__():26] System info init +2024-09-22 05:15:51,850 DEBUG HandlerThread:1229619 [system_info.py:__init__():41] System info init done +2024-09-22 05:15:51,850 INFO HandlerThread:1229619 [system_monitor.py:start():194] Starting system monitor +2024-09-22 05:15:51,851 INFO SystemMonitor:1229619 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-22 05:15:51,851 INFO HandlerThread:1229619 [system_monitor.py:probe():214] Collecting system info +2024-09-22 05:15:51,851 INFO SystemMonitor:1229619 [interfaces.py:start():188] Started cpu monitoring +2024-09-22 05:15:51,852 INFO SystemMonitor:1229619 [interfaces.py:start():188] Started disk monitoring +2024-09-22 05:15:51,853 INFO SystemMonitor:1229619 [interfaces.py:start():188] Started gpu monitoring +2024-09-22 05:15:51,853 INFO SystemMonitor:1229619 [interfaces.py:start():188] Started memory monitoring +2024-09-22 05:15:51,854 INFO SystemMonitor:1229619 [interfaces.py:start():188] Started network monitoring +2024-09-22 05:15:51,933 DEBUG HandlerThread:1229619 [system_info.py:probe():152] Probing system +2024-09-22 05:15:51,937 DEBUG HandlerThread:1229619 [system_info.py:_probe_git():137] Probing git +2024-09-22 05:15:51,951 DEBUG HandlerThread:1229619 [system_info.py:_probe_git():145] Probing git done +2024-09-22 05:15:51,951 DEBUG HandlerThread:1229619 [system_info.py:probe():200] Probing system done +2024-09-22 05:15:51,951 DEBUG HandlerThread:1229619 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-09-21T21:15:51.934138', 'startedAt': '2024-09-21T21:15:51.132992', 'docker': None, 'cuda': None, 'args': ('--local_rank=0', '--model_name_or_path', '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', '--train_datasets', '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', '--output_dir', '../outputs/dpo_tiv2t_10k_baseline', '--train_data_files', 'extracted_preference_10k_washed.json', '--train_template', 'NExTQA_preference', '--train_split', 'train', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--gradient_accumulation_steps', '1'), 'state': 'running', 'program': '-m align_anything.trainers.tiv_to_t.dpo', 'codePathLocal': None, 'git': {'remote': 'git@github.com-hantao:htlou/align-anything.git', 'commit': '9eff99b0ee34f7265b8a2e1824650634c898f654'}, 'email': '2200017789@stu.pku.edu.cn', 'root': '/aifs4su/yaodong/projects/hantao/dev_cham/align-anything', 'host': 'dgx-081', 'username': 'yangyaodong', 'executable': '/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 2100.4980491071433, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3792.33, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.996, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 1758.8549690246582, 'used': 58.09828567504883}}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2015.5615730285645}} +2024-09-22 05:15:51,951 INFO HandlerThread:1229619 [system_monitor.py:probe():224] Finished collecting system info +2024-09-22 05:15:51,951 INFO HandlerThread:1229619 [system_monitor.py:probe():227] Publishing system info +2024-09-22 05:15:51,952 DEBUG HandlerThread:1229619 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-09-22 05:15:52,598 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/conda-environment.yaml +2024-09-22 05:15:56,601 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/conda-environment.yaml +2024-09-22 05:15:56,727 DEBUG HandlerThread:1229619 [system_info.py:_save_conda():224] Saving conda packages done +2024-09-22 05:15:56,729 INFO HandlerThread:1229619 [system_monitor.py:probe():229] Finished publishing system info +2024-09-22 05:15:56,741 DEBUG SenderThread:1229619 [sender.py:send():379] send: files +2024-09-22 05:15:56,742 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-09-22 05:15:56,742 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:15:57,166 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: python_packages +2024-09-22 05:15:57,166 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: python_packages +2024-09-22 05:15:57,167 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:15:57,167 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:15:57,168 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:15:57,396 DEBUG SenderThread:1229619 [sender.py:send():379] send: telemetry +2024-09-22 05:15:57,550 INFO wandb-upload_0:1229619 [upload_job.py:push():130] Uploaded file /tmp/tmp4q6smt_1wandb/qkpwnwnl-wandb-metadata.json +2024-09-22 05:15:57,602 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-metadata.json +2024-09-22 05:15:57,602 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/requirements.txt +2024-09-22 05:15:57,603 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:15:58,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:15:59,167 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:15:59,604 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:16:00,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:01,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:02,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:02,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:03,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:04,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:05,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:06,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:07,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:07,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:08,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:09,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:10,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:11,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:11,613 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:16:12,167 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:16:12,167 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:16:12,210 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:12,448 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:13,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:14,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:15,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:16,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:16,742 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:16:16,742 DEBUG SenderThread:1229619 [sender.py:send():379] send: telemetry +2024-09-22 05:16:17,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:17,618 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:16:18,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:18,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:19,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:20,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:21,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:23,171 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:23,624 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/config.yaml +2024-09-22 05:16:28,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:30,496 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:30,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:16:30,497 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:16:31,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:32,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:33,038 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:16:33,039 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:16:33,039 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:16:33,042 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:16:33,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:33,633 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:16:34,043 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:34,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:35,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:35,635 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:16:36,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:37,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:38,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:39,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:39,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:40,497 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:41,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:44,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:49,498 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:51,855 DEBUG SystemMonitor:1229619 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-22 05:16:51,858 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:16:54,859 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:16:55,085 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:16:55,085 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:55,085 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:16:56,083 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:56,634 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:16:56,635 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:16:56,635 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:16:56,637 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:16:56,655 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:16:57,083 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:57,656 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:16:58,083 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:16:59,083 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:00,083 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:00,084 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:05,084 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:10,084 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:15,085 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:17,820 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:17,820 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:17:17,821 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:17:18,818 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:19,290 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:17:19,291 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:17:19,291 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:17:19,293 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:17:19,691 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:17:19,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:20,294 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:20,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:21,693 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:17:21,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:21,859 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:17:22,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:23,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:24,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:25,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:25,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:26,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:27,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:28,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:29,820 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:30,820 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:31,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:31,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:32,817 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:17:32,818 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:17:32,862 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:33,819 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:34,820 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:37,104 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:38,765 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:39,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:40,254 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:17:40,255 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:17:40,255 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:17:40,257 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:17:40,709 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:17:40,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:41,710 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:17:41,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:42,257 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:42,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:43,765 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:44,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:45,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:46,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:47,765 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:47,765 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:47,817 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:17:47,818 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:17:48,764 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:51,861 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:17:52,862 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:17:53,363 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:54,363 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:54,850 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:17:54,851 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:17:54,851 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:17:54,852 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:17:55,363 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:55,772 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:17:55,772 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:17:56,363 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:57,364 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:17:58,853 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:03,854 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:05,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:05,386 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:18:05,386 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:18:06,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:06,894 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:18:06,895 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:18:06,895 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:18:06,897 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:18:07,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:07,785 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:18:07,785 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:18:08,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:08,898 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:09,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:10,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:11,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:12,386 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:13,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:14,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:14,385 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:19,386 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:21,862 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:18:24,863 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:29,864 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:33,355 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:33,356 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:18:33,356 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:18:34,354 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:34,762 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:18:34,763 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:18:34,763 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:18:34,765 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:18:34,812 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:18:35,354 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:35,766 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:35,813 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:18:36,354 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:37,354 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:40,767 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:45,767 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:50,768 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:18:51,864 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:18:51,927 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:51,928 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:18:51,928 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:18:52,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:53,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:18:53,311 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:18:53,311 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:18:53,312 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:18:53,830 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:18:53,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:54,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:18:55,832 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:18:56,313 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:01,314 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:03,854 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:04,854 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:05,452 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:19:05,452 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:19:05,453 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:19:05,455 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:19:05,842 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:19:05,854 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:06,456 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:06,854 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:06,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:19:06,927 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:19:07,845 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:19:07,855 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:12,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:17,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:21,865 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:19:22,866 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:24,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:24,155 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:19:24,155 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:19:25,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:25,942 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:19:25,943 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:19:25,943 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:19:25,944 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:19:26,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:26,876 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:19:27,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:27,877 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:19:27,945 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:28,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:32,945 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:37,946 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:38,525 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:39,153 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:19:39,153 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:19:39,524 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:40,326 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:19:40,327 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:19:40,327 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:19:40,329 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:19:40,524 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:40,904 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:19:41,524 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:41,904 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:19:42,524 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:43,330 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:48,330 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:51,868 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:19:53,869 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:58,869 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:19:59,418 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:19:59,418 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:19:59,418 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:20:00,416 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:01,416 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:01,722 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:20:01,723 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:20:01,723 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:20:01,724 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:20:01,937 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:20:02,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:03,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:03,939 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:20:04,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:04,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:05,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:06,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:07,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:08,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:09,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:10,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:10,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:15,417 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:20,418 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:21,868 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:20:23,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:20:23,517 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:20:23,558 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:24,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:25,278 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:20:25,279 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:20:25,279 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:20:25,281 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:20:25,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:25,975 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:20:26,281 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:26,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:27,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:27,977 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:20:28,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:29,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:30,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:31,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:31,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:32,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:33,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:34,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:35,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:36,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:37,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:37,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:38,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:20:38,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:38,516 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:20:39,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:40,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:41,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:42,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:42,808 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:47,809 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:49,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:50,378 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:51,094 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:20:51,095 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:20:51,095 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:20:51,097 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:20:51,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:51,870 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:20:51,998 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:20:52,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:52,871 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:53,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:53,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:20:53,516 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:20:53,999 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:20:54,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:55,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:56,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:57,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:58,379 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:20:58,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:20:59,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:00,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:01,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:02,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:03,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:08,380 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:09,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:21:09,516 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:21:09,558 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:10,515 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:11,066 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:21:11,067 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:21:11,067 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:21:11,069 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:21:11,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:12,017 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:21:12,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:13,516 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:14,019 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:21:14,069 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:19,070 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:21,873 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:21:24,874 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:29,875 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:34,875 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:39,876 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:44,876 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:48,614 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:21:48,615 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:48,615 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:21:49,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:49,923 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:50,038 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:21:50,039 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:21:50,039 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:21:50,041 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:21:50,047 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:21:50,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:51,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:51,873 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:21:52,050 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:21:52,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:53,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:54,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:55,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:21:55,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:21:56,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:00,614 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:05,614 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:05,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:22:05,701 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:22:05,742 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:06,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:07,038 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:22:07,039 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:22:07,039 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:22:07,040 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:22:07,062 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:22:07,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:08,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:09,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:10,065 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:22:10,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:10,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:11,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:12,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:13,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:14,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:15,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:16,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:16,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:17,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:18,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:19,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:20,700 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:22:20,700 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:22:20,742 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:21,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:21,874 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:22:21,874 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:22,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:23,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:24,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:25,558 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:22:25,559 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:22:25,559 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:22:25,561 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:22:25,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:26,077 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:22:26,702 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:27,701 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:27,702 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:28,079 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:22:32,702 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:37,166 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:37,167 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:22:37,167 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:22:38,165 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:38,357 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:38,508 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:22:38,509 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:22:38,509 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:22:38,510 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:22:39,088 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:22:39,165 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:40,089 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:22:43,511 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:48,512 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:51,875 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:22:53,501 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:53,501 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:22:53,502 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:22:53,798 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:22:54,501 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:55,270 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:22:55,271 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:22:55,271 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:22:55,272 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:22:55,500 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:56,103 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:22:56,501 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:22:58,105 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:22:59,273 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:04,274 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:09,274 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:14,275 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:19,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:19,181 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:23:19,181 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:23:19,481 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:20,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:20,562 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:23:20,563 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:23:20,563 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:23:20,565 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:23:21,125 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:23:21,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:21,877 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:23:22,126 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:23:22,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:23,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:24,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:25,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:25,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:26,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:27,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:28,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:30,180 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:35,181 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:40,181 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:45,182 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:50,182 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:51,878 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:23:55,879 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:23:57,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:23:57,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:57,933 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:23:58,931 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:23:59,406 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:23:59,407 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:23:59,408 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:23:59,409 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:23:59,931 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:00,196 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:24:00,931 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:01,410 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:01,931 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:02,197 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:24:02,932 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:06,932 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:11,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:16,934 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:16,953 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:16,954 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:24:16,954 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:24:17,953 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:18,574 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:24:18,575 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:24:18,575 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:24:18,576 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:24:18,953 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:19,227 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:24:20,228 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:24:21,879 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:24:22,880 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:27,881 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:32,882 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:37,882 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:41,784 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:24:41,784 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:41,784 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:24:42,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:43,218 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:24:43,775 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:24:43,775 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:24:43,775 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:43,776 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:24:43,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:44,266 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:24:44,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:45,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:46,268 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:24:46,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:47,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:48,782 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:48,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:49,784 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:50,784 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:51,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:51,882 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:24:53,219 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:53,883 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:24:54,219 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:54,650 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:24:54,651 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:24:54,651 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:24:54,653 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:24:55,220 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:24:55,277 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:24:56,278 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:24:59,654 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:04,655 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:09,655 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:10,102 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:10,102 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:25:10,102 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:25:11,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:11,494 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:25:11,495 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:25:11,495 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:25:11,496 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:25:12,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:12,293 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:25:13,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:14,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:14,295 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:25:15,100 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:15,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:16,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:17,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:18,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:20,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:21,882 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:25:25,884 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:26,486 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:26,486 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:25:26,487 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:25:27,485 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:27,894 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:25:27,895 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:25:27,895 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:25:27,897 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:25:28,323 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:25:28,485 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:29,486 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:30,324 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:25:30,485 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:30,898 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:35,898 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:40,899 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:43,753 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:43,754 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:25:43,754 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:25:44,752 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:45,290 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:25:45,291 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:25:45,291 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:25:45,293 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:25:45,352 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:25:45,753 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:46,293 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:46,753 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:47,753 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:25:48,355 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:25:51,294 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:25:51,884 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:25:56,885 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:01,886 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:06,886 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:07,492 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:07,493 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:26:07,493 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:26:08,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:09,000 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:26:09,000 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:26:09,001 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:26:09,003 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:26:09,392 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:26:09,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:10,393 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:26:10,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:11,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:12,004 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:12,394 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:26:12,492 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:13,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:14,492 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:17,493 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:21,887 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:26:22,888 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:27,888 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:31,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:31,141 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:26:31,141 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:26:32,139 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:32,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:26:32,551 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:26:32,551 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:26:32,553 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:26:33,139 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:33,450 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:26:33,553 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:34,139 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:34,451 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:26:35,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:36,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:37,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:38,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:39,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:39,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:40,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:41,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:42,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:43,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:44,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:45,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:50,140 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:26:50,535 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:50,535 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:26:50,535 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:26:51,534 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:51,887 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:26:52,082 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:26:52,083 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:26:52,083 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:26:52,084 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:26:52,469 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:26:52,534 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:53,534 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:26:54,471 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:26:56,085 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:01,086 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:06,086 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:06,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:06,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:27:06,454 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:27:07,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:07,830 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:27:07,831 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:27:07,831 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:27:07,832 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:27:08,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:08,484 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:27:09,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:09,485 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:27:10,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:11,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:11,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:16,453 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:21,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:21,889 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:27:26,890 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:29,435 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:27:29,435 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:27:29,478 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:30,434 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:30,826 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:27:30,827 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:27:30,827 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:27:30,829 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:27:31,434 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:31,505 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:27:31,505 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:27:32,434 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:32,829 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:37,830 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:42,830 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:47,831 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:50,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:27:50,491 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:50,492 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:27:51,489 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:51,890 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:27:51,950 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:27:51,951 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:27:51,951 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:27:51,952 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:27:52,489 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:52,540 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:27:52,953 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:27:53,489 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:53,541 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:27:54,489 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:27:57,953 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:02,954 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:07,955 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:12,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:12,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:28:12,268 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:28:13,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:13,513 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:13,774 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:28:13,775 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:28:13,775 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:28:13,777 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:28:14,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:14,578 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:28:15,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:15,579 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:28:16,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:17,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:18,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:19,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:19,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:20,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:21,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:21,891 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:28:22,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:23,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:24,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:25,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:25,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:26,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:27,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:28:27,267 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:28:27,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:28,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:29,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:30,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:30,528 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:31,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:32,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:33,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:33,718 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:28:33,719 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:28:33,719 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:28:33,721 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:28:34,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:34,597 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:28:35,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:35,611 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:28:35,721 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:36,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:37,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:38,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:39,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:40,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:41,267 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:46,268 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:51,269 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:51,893 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:28:56,894 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:28:57,694 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:57,695 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:28:57,695 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:28:58,693 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:28:59,302 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:28:59,303 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:28:59,303 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:28:59,304 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:28:59,648 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:28:59,693 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:00,693 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:01,650 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:29:01,693 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:02,305 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:02,694 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:03,694 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:07,694 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:12,695 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:13,024 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:29:13,025 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:13,025 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:29:14,023 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:14,582 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:29:14,583 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:29:14,583 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:29:14,584 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:29:14,672 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:29:15,023 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:15,687 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:29:16,023 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:17,023 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:18,024 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:18,024 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:21,894 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:29:23,895 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:28,896 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:33,896 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:37,455 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:29:37,455 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:37,455 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:29:38,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:38,752 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:29:38,752 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:29:38,752 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:29:38,753 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:29:39,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:39,708 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:29:39,708 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:29:39,754 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:40,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:41,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:42,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:43,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:44,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:45,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:45,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:46,454 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:29:50,455 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:29:51,897 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:29:55,898 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:00,899 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:05,899 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:10,900 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:15,900 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:16,311 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:16,312 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:30:16,312 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:30:17,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:17,738 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:30:17,739 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:30:17,739 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:30:17,740 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:30:17,744 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:30:18,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:19,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:19,746 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:30:20,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:21,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:21,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:21,897 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:30:22,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:23,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:24,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:25,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:26,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:26,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:31,310 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:32,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:32,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:30:32,124 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:30:33,122 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:33,570 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:30:33,571 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:30:33,571 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:30:33,573 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:30:33,764 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:30:34,122 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:35,122 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:35,766 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:30:36,122 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:36,573 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:37,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:38,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:39,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:40,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:41,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:42,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:42,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:43,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:44,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:45,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:46,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:47,122 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:30:47,122 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:30:47,166 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:47,366 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:48,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:49,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:50,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:51,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:51,374 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:30:51,375 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:30:51,375 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:30:51,376 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:30:51,785 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:30:51,899 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:30:52,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:53,123 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:30:53,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:53,787 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:30:54,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:30:58,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:03,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:08,124 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:08,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:08,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:31:08,200 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:31:09,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:09,514 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:31:09,515 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:31:09,515 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:31:09,516 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:31:09,805 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:31:10,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:11,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:11,807 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:31:12,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:13,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:13,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:14,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:15,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:16,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:17,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:18,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:19,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:19,199 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:21,901 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:31:24,902 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:29,903 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:34,904 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:39,904 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:44,905 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:47,153 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:47,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:31:47,154 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:31:48,152 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:48,602 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:31:48,603 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:31:48,604 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:31:48,605 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:31:48,843 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:31:49,152 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:49,844 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:31:50,152 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:50,606 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:31:51,152 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:51,902 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:31:52,153 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:53,157 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:31:56,154 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:01,155 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:03,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:03,710 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:32:03,710 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:32:04,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:05,210 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:32:05,211 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:32:05,211 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:32:05,212 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:32:05,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:05,862 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:32:06,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:06,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:07,710 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:07,864 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:32:08,710 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:09,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:10,709 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:11,710 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:16,710 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:19,093 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:19,093 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:32:19,094 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:32:20,092 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:20,590 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:32:20,591 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:32:20,591 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:32:20,592 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:32:20,877 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:32:21,092 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:21,878 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:32:21,903 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:32:21,904 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:22,092 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:26,904 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:31,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:32,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:32,552 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:32,938 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:32:32,939 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:32:32,939 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:32:32,940 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:32:33,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:33,893 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:32:33,893 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:32:34,092 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:32:34,092 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:32:34,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:35,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:36,551 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:37,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:38,550 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:38,551 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:43,551 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:48,552 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:51,905 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:32:52,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:32:52,644 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:32:52,686 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:53,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:53,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:32:54,030 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:32:54,031 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:32:54,031 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:32:54,032 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:32:54,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:54,916 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:32:55,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:55,917 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:32:56,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:57,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:58,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:59,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:32:59,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:00,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:01,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:02,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:03,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:04,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:05,643 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:05,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:06,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:07,642 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:33:07,642 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:33:07,686 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:08,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:09,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:10,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:10,918 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:11,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:12,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:13,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:14,534 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:33:14,534 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:33:14,534 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:33:14,535 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:33:14,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:14,938 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:33:15,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:15,939 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:33:16,536 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:16,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:17,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:18,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:19,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:20,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:21,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:21,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:21,908 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:33:22,642 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:33:22,643 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:33:22,686 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:23,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:24,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:25,646 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:26,646 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:26,909 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:27,590 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:33:27,591 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:33:27,591 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:33:27,592 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:33:27,646 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:33:27,952 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:33:29,954 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:33:32,644 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:37,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:42,645 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:47,646 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:51,908 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:33:52,909 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:33:57,909 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:02,910 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:07,910 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:10,138 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:34:10,139 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:10,139 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:34:11,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:11,590 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:34:11,591 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:34:11,591 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:34:11,592 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:34:11,996 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:34:12,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:13,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:13,593 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:13,998 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:34:14,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:15,138 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:16,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:17,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:18,138 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:19,137 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:19,138 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:21,909 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:34:24,911 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:29,911 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:33,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:34:33,170 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:34:33,210 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:34,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:34,886 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:34:34,887 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:34:34,887 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:34:34,889 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:34:35,018 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:34:35,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:35,889 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:36,019 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:34:40,890 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:45,890 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:50,891 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:51,913 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:34:52,285 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:52,285 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:34:52,285 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:34:53,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:54,250 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:34:54,251 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:34:54,251 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:34:54,252 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:34:54,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:55,039 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:34:55,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:56,040 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:34:56,253 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:34:56,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:57,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:58,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:34:59,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:00,283 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:01,285 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:06,285 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:08,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:35:08,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:08,400 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:35:09,399 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:10,006 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:35:10,007 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:35:10,007 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:35:10,008 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:35:10,053 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:35:10,399 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:11,399 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:12,009 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:12,055 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:35:12,399 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:13,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:14,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:15,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:16,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:17,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:17,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:18,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:19,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:20,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:21,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:21,912 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:35:22,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:22,913 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:23,401 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:35:23,401 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:23,401 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:35:24,400 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:28,593 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:33,593 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:38,531 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:35:38,531 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:35:38,574 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:38,738 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:39,530 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:40,531 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:41,006 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:35:41,007 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:35:41,007 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:35:41,009 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:35:41,091 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:35:41,531 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:42,092 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:35:42,531 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:44,009 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:49,010 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:51,914 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:35:54,915 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:35:55,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:55,340 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:35:55,340 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:35:56,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:57,210 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:35:57,211 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:35:57,211 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:35:57,213 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:35:57,338 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:58,124 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:35:58,338 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:35:59,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:00,126 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:36:00,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:00,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:01,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:02,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:03,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:04,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:05,338 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:05,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:06,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:07,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:08,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:10,339 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:15,340 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:15,845 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:15,846 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:36:15,846 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:36:16,844 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:17,390 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:36:17,391 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:36:17,391 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:36:17,393 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:36:17,844 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:18,156 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:36:18,845 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:19,845 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:20,158 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:36:20,394 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:21,915 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:36:25,916 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:26,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:27,168 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:28,018 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:36:28,021 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:36:28,021 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:36:28,023 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:36:28,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:28,180 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:36:29,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:30,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:30,181 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:36:30,845 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:36:30,846 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:36:31,103 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:31,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:32,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:33,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:34,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:35,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:36,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:36,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:37,169 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:41,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:46,170 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:51,171 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:51,917 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:36:55,129 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:55,129 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:36:55,129 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:36:56,127 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:56,238 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:36:56,352 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:36:56,353 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:36:56,353 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:36:56,354 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:36:57,128 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:36:57,233 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:36:58,234 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:37:01,355 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:06,355 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:11,356 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:16,356 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:21,357 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:21,919 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:37:22,884 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:37:22,884 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:37:22,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:23,883 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:24,778 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:37:24,779 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:37:24,779 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:37:24,781 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:37:24,883 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:25,289 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:37:25,884 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:26,290 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:37:26,781 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:26,884 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:31,782 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:36,782 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:41,783 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:46,784 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:51,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:37:51,193 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:37:51,234 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:51,921 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:37:51,923 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:37:52,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:52,902 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:37:52,903 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:37:52,903 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:37:52,904 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:37:53,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:53,328 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:37:54,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:54,329 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:37:55,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:56,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:57,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:37:57,193 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:02,194 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:07,194 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:12,195 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:14,330 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:14,330 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:38:14,330 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:38:15,329 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:15,862 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:38:15,863 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:38:15,863 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:38:15,865 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:38:16,329 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:16,378 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:38:17,865 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:18,380 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:38:21,925 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:38:22,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:27,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:30,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:38:30,212 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:38:30,254 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:31,211 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:31,610 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:38:31,611 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:38:31,611 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:38:31,613 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:38:32,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:32,407 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:38:33,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:33,614 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:34,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:34,408 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:38:35,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:36,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:37,212 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:38,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:39,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:39,213 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:44,214 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:45,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:45,441 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:38:45,442 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:38:46,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:47,078 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:38:47,079 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:38:47,079 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:38:47,080 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:38:47,421 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:38:47,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:48,422 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:38:48,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:49,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:50,081 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:38:50,441 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:51,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:51,925 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:38:52,440 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:38:55,926 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:00,855 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:39:00,856 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:39:00,898 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:01,099 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:01,855 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:02,718 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:39:02,719 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:39:02,719 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:39:02,721 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:39:02,855 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:03,437 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:39:04,438 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:39:06,722 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:11,722 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:16,723 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:21,723 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:21,927 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:39:26,928 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:31,929 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:36,930 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:40,000 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:40,000 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:39:40,001 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:39:40,998 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:41,430 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:39:41,431 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:39:41,432 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:39:41,433 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:39:41,473 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:39:41,998 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:42,433 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:42,998 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:43,998 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:44,476 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:39:44,999 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:39:47,999 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:51,931 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:39:53,932 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:39:58,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:00,101 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:40:00,102 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:40:00,146 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:01,100 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:01,554 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:40:01,555 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:40:01,555 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:40:01,556 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:40:02,100 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:02,492 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:40:03,100 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:04,494 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:40:04,557 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:09,558 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:14,558 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:19,559 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:21,932 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:40:24,933 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:28,523 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:40:28,523 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:40:28,566 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:29,522 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:29,974 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:30,014 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:40:30,015 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:40:30,015 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:40:30,017 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:40:30,522 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:30,529 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:40:31,522 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:32,522 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:32,531 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:40:35,017 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:40,018 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:42,354 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:43,353 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:43,522 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:40:43,523 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:40:43,765 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:40:43,766 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:40:43,766 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:40:43,768 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:40:44,353 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:40:44,543 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:40:45,768 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:46,545 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:40:50,769 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:40:51,934 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:40:55,935 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:00,936 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:02,792 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:41:02,793 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:41:02,834 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:03,796 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:04,118 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:41:04,119 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:41:04,119 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:41:04,121 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:41:04,561 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:41:04,796 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:05,796 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:06,121 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:06,563 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:41:06,796 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:07,796 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:11,793 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:16,793 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:21,794 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:21,936 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:41:26,937 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:31,938 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:32,413 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:41:32,413 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:32,413 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:41:33,412 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:33,818 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:41:33,819 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:41:33,819 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:41:33,821 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:41:34,411 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:34,588 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:41:35,589 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:41:37,822 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:42,822 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:46,825 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:47,411 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:41:47,412 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:41:47,823 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:48,340 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:48,382 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:41:48,383 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:41:48,383 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:41:48,384 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:41:48,602 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:41:48,824 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:49,602 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:41:49,824 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:41:51,939 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:41:53,940 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:41:58,941 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:03,941 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:08,942 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:09,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:09,661 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:42:09,662 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:42:10,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:11,114 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:42:11,115 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:42:11,115 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:42:11,117 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:42:11,622 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:42:11,622 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:42:11,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:12,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:13,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:14,117 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:14,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:15,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:19,660 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:21,940 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:42:24,941 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:29,942 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:34,352 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:42:34,352 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:34,352 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:42:35,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:35,613 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:35,718 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:42:35,719 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:42:35,719 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:42:35,721 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:42:36,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:36,644 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:42:37,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:37,645 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:42:38,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:39,351 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:40,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:41,350 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:41,351 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:42,351 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:42:46,351 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:51,351 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:42:51,942 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:42:56,943 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:01,944 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:06,944 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:11,374 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:11,375 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: stop_status +2024-09-22 05:43:11,375 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: stop_status +2024-09-22 05:43:12,372 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:12,642 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:12,734 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: partial_history +2024-09-22 05:43:12,735 DEBUG SenderThread:1229619 [sender.py:send():379] send: history +2024-09-22 05:43:12,735 DEBUG SenderThread:1229619 [sender.py:send_request():406] send_request: summary_record +2024-09-22 05:43:12,737 INFO SenderThread:1229619 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-22 05:43:13,372 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:13,685 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/wandb-summary.json +2024-09-22 05:43:13,685 INFO Thread-12 :1229619 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_10k_baseline/wandb/run-20240922_051551-1efs7bbc/files/output.log +2024-09-22 05:43:14,372 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:15,372 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:16,372 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:17,373 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:18,373 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:18,374 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:19,373 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:20,373 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-22 05:43:21,946 DEBUG SenderThread:1229619 [sender.py:send():379] send: stats +2024-09-22 05:43:23,947 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report +2024-09-22 05:43:28,947 DEBUG HandlerThread:1229619 [handler.py:handle_request():158] handle_request: status_report