tokenizer_config.json

{
  "add_prefix_space": false,
  "added_tokens_decoder": {
    "0": {
      "content": ">>TITLE<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "1": {
      "content": ">>ABSTRACT<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "2": {
      "content": ">>INTRODUCTION<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "3": {
      "content": ">>SUMMARY<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "4": {
      "content": ">>COMMENT<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "5": {
      "content": ">>ANSWER<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "6": {
      "content": ">>QUESTION<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "7": {
      "content": ">>DOMAIN<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "8": {
      "content": ">>PREFIX<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "9": {
      "content": ">>SUFFIX<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "10": {
      "content": ">>MIDDLE<<",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    },
    "11": {
      "content": "<|endoftext|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false,
      "special": true
    }
  },
  "additional_special_tokens": [
    ">>TITLE<<",
    ">>ABSTRACT<<",
    ">>INTRODUCTION<<",
    ">>SUMMARY<<",
    ">>COMMENT<<",
    ">>ANSWER<<",
    ">>QUESTION<<",
    ">>DOMAIN<<",
    ">>PREFIX<<",
    ">>SUFFIX<<",
    ">>MIDDLE<<"
  ],
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'User: ' + content + '\nFalcon:' }}{% elif message['role'] == 'assistant' %}{{ content + '\n' }}{% endif %}{% endfor %}",
  "clean_up_tokenization_spaces": true,
  "eos_token": "<|endoftext|>",
  "model_input_names": [
    "input_ids",
    "attention_mask"
  ],
  "model_max_length": 2048,
  "pad_token": "<|endoftext|>",
  "padding_side": "right",
  "split_special_tokens": false,
  "tokenizer_class": "PreTrainedTokenizerFast"
}