code.py

import numpy as np
import pandas as pd
import torch
from transformers import XLNetTokenizer, XLNetForSequenceClassification, Trainer, TrainingArguments
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, matthews_corrcoef, confusion_matrix

# Mount Google Drive
from google.colab import drive
drive.mount('/content/drive', force_remount=True)

# Path to your Excel file in Google Drive
file_path = '/content/drive/My Drive/filtered_data.xlsx'

# Read the Excel file into a pandas DataFrame
df = pd.read_excel(file_path)

# Selecting only the necessary columns
selected_columns = ['Short Description', 'Severity Label', 'Project']
new_df = df[selected_columns].copy()

# Exclude bug reports with 'normal' severity
filtered_df = new_df[new_df['Severity Label'] != 'normal']

# Define mapping for label conversion
severity_mapping = {
    'blocker': 'severe',
    'critical': 'severe',
    'major': 'severe',
    'trivial': 'non-severe',
    'minor': 'non-severe'
}

# Replace severity labels according to the mapping
filtered_df.loc[:, 'Severity Label'] = filtered_df['Severity Label'].map(severity_mapping)

# Mapping string labels to numeric representations
label_mapping = {'non-severe': 0, 'severe': 1}
filtered_df.loc[:, 'Severity Label'] = filtered_df['Severity Label'].map(label_mapping)


####

# Initialize XLNet tokenizer and model
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')


# Define parameters
max_len = 100  # Max sequence length
batch_size = 32
epochs = 5

# Initialize evaluation results dictionary
evaluation_results = {}

# Iterate through each unique project as the test set
for test_project in filtered_df['Project'].unique():

    # Reinitialize the model for each test project
    model = XLNetForSequenceClassification.from_pretrained('xlnet-base-cased', num_labels=2)  # Define num_labels for binary classification

    # Select data for the current test project
    test_data = filtered_df[filtered_df['Project'] == test_project]
    train_data = filtered_df[filtered_df['Project'] != test_project]

    # Split train and test data
    train_texts = train_data['Short Description'].tolist()
    train_labels = train_data['Severity Label'].tolist()
    test_texts = test_data['Short Description'].tolist()
    test_labels = test_data['Severity Label'].tolist()

    # Tokenize train and test data
    train_encodings = tokenizer(train_texts, truncation=True, padding='max_length', max_length=max_len)
    test_encodings = tokenizer(test_texts, truncation=True, padding='max_length', max_length=max_len)

    # Create PyTorch datasets
    class CustomDataset(torch.utils.data.Dataset):
        def __init__(self, encodings, labels):
            self.encodings = encodings
            self.labels = labels

        def __getitem__(self, idx):
            item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
            item['labels'] = torch.tensor(self.labels[idx])
            return item

        def __len__(self):
            return len(self.labels)

    train_dataset = CustomDataset(train_encodings, train_labels)
    test_dataset = CustomDataset(test_encodings, test_labels)

    # Define training arguments
    training_args = TrainingArguments(
        output_dir='./results',  # output directory
        num_train_epochs=epochs,  # total number of training epochs
        per_device_train_batch_size=batch_size,  # batch size per device during training
        per_device_eval_batch_size=batch_size,   # batch size for evaluation
        warmup_steps=500,  # number of warmup steps for learning rate scheduler
        weight_decay=0.01,  # strength of weight decay
        logging_dir='./logs',  # directory for storing logs
    )

    # Define trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
    )

    # Train the model
    trainer.train()

    # Save the model to Google Drive
    model_save_path = '/content/drive/My Drive/XLNet_model_project_{}.pt'.format(test_project)
    torch.save(model.state_dict(), model_save_path)
    print(f"Model saved to '{model_save_path}'")

    # Evaluate the model
    predictions = trainer.predict(test_dataset)
    preds = np.argmax(predictions.predictions, axis=1)

    # Calculate evaluation metrics
    accuracy = accuracy_score(test_labels, preds)
    precision = precision_score(test_labels, preds)
    recall = recall_score(test_labels, preds)
    f1 = f1_score(test_labels, preds)
    mcc = matthews_corrcoef(test_labels, preds)
    conf_matrix = confusion_matrix(test_labels, preds)

    # Store evaluation results for the current test project
    evaluation_results[test_project] = {
        'Accuracy': accuracy,
        'Precision': precision,
        'Recall': recall,
        'F1-score': f1,
        'MCC': mcc,
        'Confusion Matrix': conf_matrix
    }

# Print evaluation results for all test projects
for project, results in evaluation_results.items():
    print(f"Evaluation results for Test Project '{project}':")
    for metric, value in results.items():
        if metric != 'Confusion Matrix':
            print(f"{metric}: {value}")
        else:
            print(f"{metric}:")
            print(value)
    print("------------------------------")

# Convert evaluation results to a DataFrame
df_results = pd.DataFrame.from_dict(evaluation_results, orient='index')

# Save results to an Excel file
excel_file_name = '/content/drive/My Drive/evaluation_results_XLNet.xlsx'
df_results.to_excel(excel_file_name)
print(f"Results saved to '{excel_file_name}'")