{ | |
"n_layers": 32, | |
"dim": 4096, | |
"hidden_dim": 14336, | |
"n_heads": 32, | |
"head_dim": 128, | |
"n_kv_heads": 8, | |
"norm_eps": 1e-05, | |
"vocab_size": 32000, | |
"sliding_window": 4096 | |
} |
{ | |
"n_layers": 32, | |
"dim": 4096, | |
"hidden_dim": 14336, | |
"n_heads": 32, | |
"head_dim": 128, | |
"n_kv_heads": 8, | |
"norm_eps": 1e-05, | |
"vocab_size": 32000, | |
"sliding_window": 4096 | |
} |