config.json

{
  "architectures": [
    "TokenFormerForCausalLM"
  ],
  "num_layers": 40,
  "hidden_size": 1536,
  "num_attention_heads": 16,
  "qkv_slot_num": 1536,
  "proj_slot_num": 1536,
  "ffn_slot_num": 6144,
  "seq_length": 2048,
  "max_position_embeddings": 2048,
  "pos_emb": "rotary",
  "rotary_pct": 0.25,
  "no_weight_tying": false,
  "norm": "layernorm_nonparam",
  "final_norm": "layernorm",
  "gpt_j_residual": false,
  "output_layer_parallelism": "column",
  "use_bias_in_attn_linear": false,
  "attention_config": [[["tokenformer"], 40]],
  "norm_activation_type": "l2_norm_gelu",
  "scaled_upper_triang_masked_softmax_fusion": false,
  "bias_gelu_fusion": false,
  "rope_fusion": false,
  "layernorm_fusion": false,
  "init_method": "normal", 
  "output_layer_init_method": "wang_init",
  "use_cache": true,
  "torch_dtype": "float16",
  "transformers_version": "4.36.0"
}