calculate.py

# Read the CSV data into a DataFrame
data = pd.read_csv('models.csv')

# Define the score columns
score_columns = ['Average', 'AGIEval', 'GPT4All', 'TruthfulQA', 'Bigbench']

# Function to calculate the highest combined score for a given column
def calculate_highest_combined_score(column):
    start_time = time.time()
    scores = data[column].tolist()
    models = data['Model'].tolist()
    top_combinations = {2: [], 3: [], 4: [], 5: [], 6: []}
    calculations = {2: 0, 3: 0, 4: 0, 5: 0, 6: 0}

    # Generate all unique combinations of two, three, four, and five models
    for r in range(2, 7):  # r is the combination size (2, 3, 4, 5 or 6)
        for combination in combinations(zip(scores, models), r):
            combined_score = sum(score for score, _ in combination)
            # Add the combination to the list along with its score
            top_combinations[r].append((combined_score, tuple(model for _, model in combination)))
            calculations[r] += 1
        # Sort the list in descending order by score and keep only the top three
        top_combinations[r] = sorted(top_combinations[r], key=lambda x: x[0], reverse=True)[:3]

    elapsed_time = time.time() - start_time
    return column, top_combinations, calculations, elapsed_time

# Function to be executed in parallel
def worker(column):
    return calculate_highest_combined_score(column)

if __name__ == '__main__':
    with Pool() as pool:
        results = pool.map(worker, score_columns)
    # Sort results by max_score in descending order
    sorted_results = sorted(results, key=lambda x: max(x[1][5])[0] if 5 in x[1] else 0, reverse=True)
    # Print the sorted results
    for column, top_combinations, calculations, elapsed_time in sorted_results:
        for r in range(2, 7):
            print(f"Column: {column}, Number of Models: {r}")
            for score, combination in top_combinations[r]:
                print(f"Combination: {combination}, Score: {score}")
            print(f"Calculations required: {calculations[r]}")
            print(f"Time taken: {elapsed_time:.4f} seconds")
            print()  # Add an empty line for better readability

    # Count how many times each model is mentioned
    model_mentions = Counter()
    for _, top_combinations, _, _ in sorted_results:
        for r in range(2, 7):
            for _, combination in top_combinations[r]:
                model_mentions.update(combination)

    # Print the top 5 most mentioned models
    print("Top 5 most mentioned models:")
    for model, count in model_mentions.most_common(5):
        print(f"{model}: {count} times")