danielhanchen commited on
Commit
1a5698f
1 Parent(s): d00516f

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +1 -1
config.json CHANGED
@@ -22,7 +22,7 @@
22
  "num_hidden_layers": 42,
23
  "num_key_value_heads": 8,
24
  "pad_token_id": 0,
25
- "query_pre_attn_scalar": 224,
26
  "rms_norm_eps": 1e-06,
27
  "rope_theta": 10000.0,
28
  "sliding_window": 4096,
 
22
  "num_hidden_layers": 42,
23
  "num_key_value_heads": 8,
24
  "pad_token_id": 0,
25
+ "query_pre_attn_scalar": 256,
26
  "rms_norm_eps": 1e-06,
27
  "rope_theta": 10000.0,
28
  "sliding_window": 4096,