oa-llama3-r64-a32 / special_tokens_map.json
gautierdag's picture
Upload tokenizer
7b0382d verified
{
"additional_special_tokens": [
{
"content": "[0]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[A1]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[A2]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[A3]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[B1]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[B2]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[B3]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[C1]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[C2]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[C3]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I1]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I2]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I3]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I4]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I5]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I6]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I7]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I8]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I9]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I10]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I11]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I12]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I13]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I14]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I15]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I16]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I17]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I18]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I19]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I20]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I21]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I22]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I23]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I24]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I25]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I26]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I27]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I28]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I29]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I30]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I31]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I32]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I33]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I34]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I35]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "[I36]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
],
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}