import numpy as np import gradio as gr import os import pandas as pd from datasets import load_dataset from sklearn.metrics.pairwise import cosine_similarity from datasets import Features, Value import plotly.express as px features = Features({'matn': Value('string'), 'taraf_ID': Value('string'), 'bookid_hadithid': Value('string')}) Secret_token = os.getenv('HF_token') dataset = load_dataset("FDSRashid/embed_matn", token = Secret_token) books = load_dataset('FDSRashid/Hadith_info', data_files='Books.csv', token=Secret_token)['train'].to_pandas() df = dataset["train"].to_pandas() dataset = load_dataset("FDSRashid/hadith_info", data_files = 'All_Matns.csv',token = Secret_token, features = features) matn_info = dataset['train'].to_pandas() matn_info = matn_info.drop(97550) matn_info = matn_info.drop(307206) matn_info['taraf_ID'] = matn_info['taraf_ID'].replace('KeyAbsent', -1) matn_info['taraf_ID'] = matn_info['taraf_ID'].astype(int) matn_info['Book_ID'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[0])) matn_info['Hadith Number'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[1])) matn_info = pd.merge(matn_info, books, on='Book_ID') matn_info = matn_info.reset_index() df = df.reset_index() cols_to_use = df.columns.difference(matn_info.columns) joined_df = pd.merge(matn_info,df[cols_to_use],left_index=True, right_index=True) df = joined_df.copy() taraf_max = np.max(df['taraf_ID'].unique()) def plot_similarity_score(taraf_num): taraf_df = df[df['taraf_ID']== taraf_num] taraf_df['Number'] = np.arange(len(taraf_df)) embed_taraf = taraf_df['embed'].to_list() cos_score = cosine_similarity(embed_taraf) fig = px.imshow(cos_score) matr = cos_score rows, cols = matr.shape mask = np.tril(np.ones((rows, cols), dtype=bool), k=-1) lower_triangle = matr[mask] data = lower_triangle.flatten() fig_dis = px.histogram(x = data, title = f'Similarity Distribution for Taraf {taraf_num}', labels = {'x': 'Similarity Score'}, nbins = 20, template = 'ggplot2' ) return fig, fig_dis, taraf_df[['matn', 'Number', 'Book_Name', 'Author', 'Hadith Number']] with gr.Blocks() as demo: gr.Markdown('# Semantic Similarity Visualizer') taraf_number = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1) btn = gr.Button('Submit') btn.click(fn = plot_similarity_score, inputs = [taraf_number], outputs = [gr.Plot(),gr.Plot(), gr.DataFrame(wrap=True)]) demo.launch()