File size: 2,405 Bytes
e57c362 0d20ff6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
---
language:
- en
license: apache-2.0
tags:
- text-generation-inference
- transformers
- unsloth
- mistral
- trl
base_model: unsloth/mistral-7b-v0.3-bnb-4bit
---
# Uploaded model
- **Developed by:** jingwang
- **License:** apache-2.0
- **Finetuned from model :** unsloth/mistral-7b-v0.3-bnb-4bit
This mistral model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library.
# install dependencies in google colab
```shell
!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
!pip install --no-deps xformers "trl<0.9.0" peft accelerate bitsandbytes
```
# inference
```python
from unsloth import FastLanguageModel
from typing import Dict, List, Tuple, Union, Any
import pandas
from tqdm import trange, tqdm
import torch
class FormatPrompt_QA_with_citation():
'''format prompt class'''
def __init__(self, eos_token:str='</s>') -> None:
self.inputs = ['context','question'] # required input fields
self.outputs = ['answer', 'citation'] # for training, and model inference output fields
self.eos_token = eos_token
def __call__(self, instance: Dict[str, Any]) -> str:
'''
function call operator
Args:
instance: dictionary with keys: 'question', 'answer'
Returns:
prompt: formatted prompt
'''
return self.formatting_prompt_func(instance)
def formatting_prompt_func(self, instance: dict) -> str:
'''format prompt for domain specific QA
note this is for fine-tuning pre-trained model,
if starting with instuct tuned model, use `tokenizer.apply_chat_template(messages)` instead
'''
assert all([ item in instance.keys() for item in self.inputs ]), logging.info(f"instance must have {self.inputs}!")
prompt = f"""<s> [INST] Context: {str(instance["context"])}\
Question: {str(instance["question"])} [/INST]
Answer: """
if ('answer' in instance):
if ('citation' in instance):
answer = {"answer":str(instance['answer']), "citation":str(instance['citation'])}
else:
answer = {"answer":str(instance['answer']), "citation":""}
prompt += json.dumps(answer, ensure_ascii=False) + self.eos_token # json format
else:
pass
return prompt
```
|