Upload 2 files

handler.py

@@ -1,43 +1,32 @@
-from unsloth import FastLanguageModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 import json
 
 class EndpointHandler():
   def __init__(self, path="ceiteach/chart-no-pretrain-llama31-unsloth"):
-    max_seq_length = 4096
-    dtype = None
-    load_in_4bit = True
-    model_name = "ceiteach/chart-no-pretrain-llama31-unsloth"
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name = model_name,
-        max_seq_length = max_seq_length,
-        dtype = dtype,
-        load_in_4bit = load_in_4bit,
-    )
+    model_name = path
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype = torch.float16, device_map="auto")
     self.model = model
     self.tokenizer = tokenizer
 
   def __call__(self, data):
     model = self.model
     tokenizer = self.tokenizer
-    FastLanguageModel.for_inference(model)
 
-    date = "2024-08-18"
+    date = "2024-08-19"
     previous_metric = "CPFORD"
     previous_chart_type = "line"
     instruction = """
 You are responding to an athlete who wants to view their data in a chart.
 Today's date is 2024-08-18 (YYYY-MM-DD).
-
 You have access to the following functions. Your response must use at least one of these functions.
-
 Functions: "
 - {"name":"draw","description":"Use this tool to add or update a dataset in the chart.","parameters":{"type":"OBJECT","description":"The parameters for the draw function.","properties":{"chartType":{"type":"STRING","description":"The type of chart to draw. It can be either 'line' or 'bar'. If unspecified use the previous chart type which will be provided in the prompt."},"metric":{"type":"STRING","description":"The metric to be charted. If unspecified use the previous metric which will be provided in the prompt."}},"required":["chartType","metric"]}}
 - {"name":"updateDateRange","description":"Use this tool to update the date range of the chart.","parameters":{"type":"OBJECT","description":"The parameters for the updateDateRange function.","properties":{"startDate":{"type":"STRING","description":"The start date of the chart. If unspecified use the previous start date which will be provided in the prompt."}},"required":["startDate"]}}
 - {"name":"removeDataset","description":"Use this tool to remove a dataset from the chart.","parameters":{"type":"OBJECT","description":"The parameters for the removeDataset function.","properties":{"metric":{"type":"STRING","description":"The metric to be removed. If unspecified use the previous metric which will be provided in the prompt."}},"required":["metric"]}}
 - {"name":"clearChart","description":"Use this tool to clear the chart.","parameters":{"type":"OBJECT","description":"The parameters for the clearChart function.","properties":{}}}
 "
-
 "Metrics" is a map of metric names to metric ids. The format is '<metric name>': '<metric id>'.
 Metrics: "
 'Accelerations': accelerationCount, 
@@ -280,7 +269,6 @@ Metrics: "
 'Omega 3 Fatty Acids': O3FA, 
 'Omega 3 Index': O3INDEX
 "
-
 Rules to follow: "
 - Use the "draw" tool to add or update a dataset in the chart.
 - Use the "updateDateRange" tool to update the date range of the chart.
@@ -296,7 +284,6 @@ Rules to follow: "
 - Only use the "removeDataset", "removeHighlight" or "clearChart" tools if the user has explicitly asked for it.
 - "removeDataset" will remove a particular dataset. 'clearChart' will completely clear the chart. If the user mentions an athlete or metric, you should assume they want to remove a dataset and not clear the chart.
 "
-
 If a new chart type or metric is not provided, use the previous values as we can assume the user wants to continue with the same options.
 - The previous chart type is line.
 - The previous metric is CPFORD.
@@ -304,16 +291,14 @@ If a new chart type or metric is not provided, use the previous values as we can
 
     user_input = data.pop("inputs", data)
     alpaca_prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
-
 ### Instruction:
 {instruction}
-
 ### Input:
 {user_input}
-
 ### Response:
 """
     prompt_tokenized = tokenizer(alpaca_prompt, return_tensors="pt").to("cuda")
+    model.eval()
     with torch.no_grad():
       response = tokenizer.decode(model.generate(**prompt_tokenized, max_new_tokens=128, temperature=0.01)[0], skip_special_tokens=True)
       response_content = response.split("### Response:")[1].strip()
@@ -323,7 +308,3 @@ If a new chart type or metric is not provided, use the previous values as we can
       except json.JSONDecodeError as e:
         print("Failed to parse function calls from response")
         return { "response": "Unable to generate response"}
-
-
-
-

requirements.txt

@@ -1,6 +1,3 @@
-unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git
-xformers==0.0.26
-trl==0.8.6
-peft==0.12.0
-accelerate==0.32.1
-bitsandbytes==0.43.3
+transformers==4.43.1
+bitsandbytes
+peft