from fastapi import APIRouter from datetime import datetime from datasets import load_dataset from sklearn.metrics import accuracy_score import random from .utils.evaluation import TextEvaluationRequest from .utils.emissions import tracker, clean_emissions_data, get_space_info ## add-on imports import numpy as np # Logistic REG reqs from sentence_transformers import SentenceTransformer from sklearn.preprocessing import MinMaxScaler import skops.io as sio # BERT reqs from transformers import AutoTokenizer,BertForSequenceClassification,AutoModelForSequenceClassification,Trainer, TrainingArguments,DataCollatorWithPadding from datasets import Dataset import torch router = APIRouter() DESCRIPTION = "Simple BERT classif" ROUTE = "/text" @router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION) async def evaluate_text(request: TextEvaluationRequest): """ Evaluate text classification for climate disinformation detection. Current Model: Random Baseline - Makes random predictions from the label space (0-7) - Used as a baseline for comparison """ # Get space info username, space_url = get_space_info() # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, "1_not_happening": 1, "2_not_human": 2, "3_not_bad": 3, "4_solutions_harmful_unnecessary": 4, "5_science_unreliable": 5, "6_proponents_biased": 6, "7_fossil_fuels_needed": 7 } # Load and prepare the dataset dataset = load_dataset(request.dataset_name) # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"].train_test_split(test_size=request.test_size, seed=request.test_seed) test_dataset = train_test["test"] # Start tracking emissions tracker.start() tracker.start_task("inference") #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE CODE HERE # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked. #-------------------------------------------------------------------------------------------- ######################## LOG REG ######################## # ## Models loading # # Embedding model # query_prompt_name = "s2s_query" # model = SentenceTransformer("dunzhang/stella_en_400M_v5",trust_remote_code=True).cuda() # # Pre-trained Logistic Regression model # trusted_types = ['sklearn.feature_selection._univariate_selection.f_classif'] # disp = sio.load('./tasks/logistic_regression_model.skops',trusted=trusted_types) # ## Data prep # embeddings = model.encode(list(test_dataset['quote']), prompt_name=query_prompt_name) # scaler = MinMaxScaler() # X_scaled = scaler.fit_transform(embeddings) # ## Predictions # predictions = disp.predict(X_scaled) ######################## BERT ######################## ## Model loading model = BertForSequenceClassification.from_pretrained("Oriaz/climate_change_bert_classif") tokenizer = AutoTokenizer.from_pretrained("Oriaz/climate_change_bert_classif") ## Data prep def preprocess_function(df): return tokenizer(df["quote"], truncation=True) tokenized_test = test_dataset.map(preprocess_function, batched=True) ## Modify inference model training_args = torch.load("./tasks/utils/training_args.bin") training_args.eval_strategy='no' trainer = Trainer( model=model, args=training_args, tokenizer=tokenizer ) ## prediction preds = trainer.predict(tokenized_test) predictions = np.array([np.argmax(x) for x in preds[0]]) #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE STOPS HERE #-------------------------------------------------------------------------------------------- # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy true_labels = test_dataset["label"] accuracy = accuracy_score(true_labels, predictions) # Prepare results dictionary results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": DESCRIPTION, "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data), "api_route": ROUTE, "dataset_config": { "dataset_name": request.dataset_name, "test_size": request.test_size, "test_seed": request.test_seed } } return results