aws_output_bucket: s3://panda-us-west-2/experiments/llama2.13b.wudao.sft.combine.legal.v1.0.seq2k.w16.adamw.NA100.0921.ds data_dir: null dist_load_data_barrier: false train_file: data/files/tmp.json dev_file: null test_file: null model: _target_: models.llama.LlamaForConditionalGeneration.from_pretrained use_peft: false gradient_checkpointing: true enable_flash_attention: true flash_attention_vanilla_torch: true read_tensor: _target_: data.collators.zh_instruct.TextDatasetUnifyV3 pair_file_list: null extended_vocab: null collator: _target_: data.collators.flan.FlanCollatorOverCollator collator: null max_seq_length: 2048 tokenizer: ${model_name_or_path} decoder_only: true padding: longest padding_side: right num_workers: 4 prefetch_factor: 2 do_preprocess: false model_name_or_path: /tmp/llama2.13b.wudao.sft.combine.v1.0.seq2k.w16.adamw.NA100.0803.ds/checkpoint-1750 pretrain: null exp_name: llama2.13b.wudao.sft.combine.legal.v1.0.seq2k.w16.adamw.NA100.0921.ds exp_notes: null output_dir: /tmp/${exp_name} resume: null do_train: true evaluate_during_training: false do_eval: false eval_sub_path: checkpoint-* per_gpu_train_batch_size: 2 per_gpu_eval_batch_size: 1 learning_rate: 1.0e-06 gradient_accumulation_steps: 8 weight_decay: 0.01 adam_epsilon: 1.0e-06 adam_betas: (0.9, 0.99) max_grad_norm: 1.0 num_train_epochs: 1 total_dataset_len: -1 max_steps: 0 warmup_proportion: 0 warmup_steps: 0 optimizer: null use_nvlamb: null bit_training: null logging_steps: 1 save_best: false save_steps: 250 eval_steps: 250 ddp_eval: true no_cuda: false seed: 42 local_rank: 0 fp16: true fp16_opt_level: O1 fp16_bfloat16: true prediction_cfg: metric: acc measure: 1 best_checkpoint: null best_result: null eval_forward_fn: _target_: general_util.evaluator.DiscriminatorForwardFn post_process: null fairscale_config: _target_: general_util.fsdp_utils.default_initialize fp16: ${fp16} move_grads_to_cpu: false move_params_to_cpu: false flatten_parameters: false with_lightseq: false load_lr_scheduler_states: false ds_cfg: train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size} gradient_accumulation_steps: ${gradient_accumulation_steps} optimizer: type: AdamW params: lr: ${learning_rate} betas: - 0.9 - 0.96 eps: ${adam_epsilon} weight_decay: ${weight_decay} scheduler: type: WarmupDecayLR params: total_num_steps: 1474 warmup_max_lr: ${learning_rate} warmup_num_steps: 0 warmup_type: linear gradient_clipping: ${max_grad_norm} bf16: enabled: ${fp16} zero_optimization: stage: 1 contiguous_gradients: true overlap_comm: true reduce_scatter: true reduce_bucket_size: 500000000.0 allgather_bucket_size: 500000000.0 offload_optimizer: device: cpu pin_memory: true steps_per_print: 1 summary_helper: _target_: general_util.tensorboard_helper.WandbWriter batch_index_or_keys: null outputs_index_or_keys: null n_gpu: 1 device: cuda:0 train_batch_size: 2 eval_batch_size: null world_size: 16 world_rank: null