tokenizer.json

// tokenizer.json
{
  "model_type": "llm",
  "tokenizer_type": "AIRider/Ko-Llama3-Luxia-8B-Q4_K_M-GGUF",
  "pretrained_tokenizer": "AIRider/Ko-Llama3-Luxia-8B-Q4_K_M-GGUF",
  "vocab_size": 50265,
  "max_length": 512,
  "padding": "max_length",
  "truncation": true
}