Add files using upload-large-folder tool

config.yaml

@@ -1,4 +1,4 @@
-run_name: olmo-400M-base
+run_name: olmo-400M-keloss_0.0015_base.23840
 seed: 6198
 epoch: null
 dry_run: false
@@ -366,10 +366,10 @@ eval_interval: 2384
 tokenizer:
   identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
   truncate_direction: right
-save_folder: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-base
+save_folder: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-keloss_0.0015_base.23840
 remote_save_folder: null
 canceled_check_interval: 50
-save_interval: 2384
+save_interval: 10
 save_interval_unsharded: 2384
 save_interval_ephemeral: null
 save_num_checkpoints_to_keep: 2
@@ -377,7 +377,7 @@ save_num_unsharded_checkpoints_to_keep: -1
 save_overwrite: true
 force_save_unsharded: false
 no_pre_train_checkpoint: false
-load_path: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-base/step9536-unsharded
+load_path: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-base/step23840-unsharded
 load_path_sharded_checkpointer: null
 try_load_latest_save: false
 reset_optimizer_state: false
@@ -387,11 +387,11 @@ new_style_checkpoints: null
 max_duration: 1ep
 global_train_batch_size: 1024
 device_train_batch_size: 128
-device_train_microbatch_size: 16
-device_eval_batch_size: 16
+device_train_microbatch_size: 4
+device_eval_batch_size: 4
 eval_subset_num_batches: -1
-eval_on_load: true
-device_train_grad_accum: 8
+eval_on_load: false
+device_train_grad_accum: 32
 max_grad_norm: 1.0
 max_grad_norm_ratio: null
 precision: amp_bf16
@@ -399,7 +399,7 @@ wandb:
   project: olmo-pretrain-ablation
   entity: alee6868
   group: null
-  name: olmo-400M-base
+  name: olmo-400M-keloss_0.0015_base.23840
   tags:
   - watching
   log_artifacts: false
@@ -438,10 +438,11 @@ inject_interval: null
 resus_portion: 1.0
 resus_ratio: 1.0
 data_shuffling: true
-KE_loss: false
+KE_loss: true
 sum_CE_KE_loss: true
-lambda_ke_loss: null
+lambda_ke_loss: 0.0015
 grad_ascent: false
 trainable_parameter: ''
+name_value: 0
 hf_datasets_cache_dir: null
 module_outputs_save_steps: null

model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:baa07c0cc94018b964838804f08f870aff022b78780e4d8682d146240be29a4d
+oid sha256:da22665aa908a8f73177c027c4172164c0102ca2f4940974adbb4f5b629ef8c4
 size 1754478590

train.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:413241207beb41cdb169d4c41aa66d66b93e9943539a2b355cad15b95314ec51
-size 15500
+oid sha256:4c3837341016560f4ea86049487da46847d61926b2e0d61e0d811fd880d583a5
+size 14988