last code change

app.py

@@ -8,7 +8,7 @@ from sentence_transformers import SentenceTransformer
 from searchmilvushelper import SearchTopKDocuments
 from finetuneresults import FineTuneAndRerankSearchResults
 from generationhelper import GenerateAnswer
-from formatresultshelper import CalculateScoresBasedOnAnswer
+from formatresultshelper import FormatAndScores
 from calculatescores import CalculateScores
 
 from model import generate_response
@@ -51,11 +51,19 @@ reranked_results = FineTuneAndRerankSearchResults(results_for_top10_chunks, rag_
 
 answer = GenerateAnswer(query, reranked_results.head(3), PROMPT_MODEL)
 
-completion_result = CalculateScoresBasedOnAnswer(query, reranked_results.head(1), answer, EVAL_MODEL)
+completion_result,relevant_sentence_keys,all_utilized_sentence_keys,support_keys,support_level = FormatAndScores(query, reranked_results.head(1), answer, EVAL_MODEL)
 
+
+print(relevant_sentence_keys)
+print(all_utilized_sentence_keys)
+print(support_keys)
+print(support_level)
 print(completion_result)
 
-score1, score2, score3 = CalculateScores()
+document_id = reranked_results.head(1)['doc_id'].values[0]
+extarcted_row_for_given_id = rag_extracted_data[rag_extracted_data["id"]==document_id]
+
+score1, score2, score3 = CalculateScores(relevant_sentence_keys,all_utilized_sentence_keys,support_keys,support_level,extarcted_row_for_given_id)
 
 print(score1)
 print(score2)

calculatescores.py

@@ -1,4 +1,7 @@
 import formatresultshelper
+import numpy as np
+
+from sklearn.metrics import mean_squared_error, roc_auc_score
 
 #Defined as utilized documents / retrieved documents for the query
 def compute_context_relevance(relevant_sentences, support_keys):
@@ -27,20 +30,20 @@ def compute_context_utilization(relevant_sentences, utilization_levels):
     return total_utilization_score / total_relevant_sentences
 
 
-def CalculateScores():
+def CalculateScores(relevant_sentence_keys,all_utilized_sentence_keys,support_keys,support_level,extarcted_row_for_given_id):
    #compute Context Relevance
-   contextrel = compute_context_relevance(formatresultshelper.relevant_sentence_keys, formatresultshelper.support_keys)
+   contextrel = compute_context_relevance(relevant_sentence_keys, support_keys)
    print(f"Context Relevance = {contextrel}")
 
-   contextutil = compute_context_utilization(formatresultshelper.relevant_sentence_keys, formatresultshelper.all_utilized_sentence_keys)
+   contextutil = compute_context_utilization(relevant_sentence_keys, all_utilized_sentence_keys)
    print(f"Context Utilization = {contextutil}")
 
-   compnum = np.intersect1d(formatresultshelper.support_keys, formatresultshelper.all_utilized_sentence_keys)
-   completenes = compnum.size / len(formatresultshelper.support_keys)
+   compnum = np.intersect1d(support_keys, all_utilized_sentence_keys)
+   completenes = compnum.size / len(support_keys)
    print(f"Completeness = {completenes}")
 
    #Adherence : whether all parts of response are grounded by context
-   for val in formatresultshelper.support_level:
+   for val in support_level:
      prevval = 1;
      if val == False:
        adherence = 0 * prevval
@@ -51,7 +54,21 @@ def CalculateScores():
 
    print(f"Adherence = {adherence}")
 
-   
+   context_relevance_score = extarcted_row_for_given_id['relevance_score'].values[0]
+   context_utilization_score = extarcted_row_for_given_id['utilization_score'].values[0]
+   adherence_score = float(extarcted_row_for_given_id['adherence_score'].values[0])
+   print(context_relevance_score)
+   print(context_utilization_score)
+   print(adherence_score)
+   docadherencearr = np.array([adherence_score, 0, 0])
+   adherencearr = np.array([adherence, 0, 0])
+   rmsecontextrel = mse(context_relevance_score, contextrel)
+   rmsecontextutil = mse(context_utilization_score, contextutil)
+   aucscore = roc_auc_score(docadherencearr, adherencearr)
+
+   return rmsecontextrel, rmsecontextutil, aucscore
+
+
 
 def mse(actual, predicted):
     return (actual - predicted)**2   

formatresultshelper.py

@@ -28,7 +28,7 @@ def evaluate_response_with_prompt(templete, query, documents, answer, eval_model
 
     return completion
 
-def CalculateScoresBasedOnAnswer(query, documents, answer, eval_model):
+def FormatAndScores(query, documents, answer, eval_model):
    templete= get_templet_to_calculatescores()
 
    completion_results = evaluate_response_with_prompt(templete, query,documents, answer, eval_model)
@@ -62,23 +62,20 @@ def CalculateScoresBasedOnAnswer(query, documents, answer, eval_model):
    sentence_support_information = data_json['sentence_support_information']
    all_utilized_sentence_keys = data_json['all_utilized_sentence_keys']
 
-   print(relavance_explanation)
-   print(relevant_sentence_keys)
-   print(overall_supported_explanation)
-   print(overall_supported)
-   print(sentence_support_information)
-   print(all_utilized_sentence_keys)
-
    support_keys = []
    support_level = []
    for sentence_support in sentence_support_information:
      support_keys += sentence_support['supporting_sentence_keys']
      support_level.append(sentence_support['fully_supported'])
 
-   print(support_keys)
-   print(support_level)
+   print(relavance_explanation)
+   print(relevant_sentence_keys)
+   print(overall_supported_explanation)
+   print(overall_supported)
+   print(sentence_support_information)
+   print(all_utilized_sentence_keys)
 
-   return completion_results_response
+   return completion_results_response,relevant_sentence_keys,all_utilized_sentence_keys,support_keys,support_level