apple
/

DCLM-7B

vaishaal commited on Jul 16

Commit

3959b33

•

1 Parent(s): 29da0e7

Update config.json

Files changed (1) hide show

config.json CHANGED Viewed

@@ -10,7 +10,7 @@
     "attn_seq_scalar": null,
     "attn_seq_scalar_alpha": null,
     "dim": 4096,
-    "ffn_type": "swiglu",
     "model": "open_lm_7b",
     "norm_type": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,

     "attn_seq_scalar": null,
     "attn_seq_scalar_alpha": null,
     "dim": 4096,
+    "ffn_type": "swiglu_torch",
     "model": "open_lm_7b",
     "norm_type": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,