{
  "architectures": [
    "Gemma2ForCausalLM"
  ],  
  
  "module": "keras_nlp.src.models.gemma.gemma_backbone",
  "class_name": "GemmaBackbone",
  "model_type": "gemma",
  "name": "gemma_backbone",
  "trainable": true,
  "vocabulary_size": 256000,
  "num_layers": 18,
  "num_query_heads": 8,
  "num_key_value_heads": 1,
  "hidden_dim": 2048,
  "intermediate_dim": 32768,
  "head_dim": 256,
  "layer_norm_epsilon": 1e-06,
  "dropout": 0,
  "query_head_dim_normalize": true,
  "use_post_ffw_norm": false,
  "use_post_attention_norm": false,
  "final_logit_soft_cap": null,
  "attention_logit_soft_cap": null,
  "sliding_window_size": 4096,
  "use_sliding_window_attention": false,
  "registered_name": "keras_nlp>GemmaBackbone"
}