feat: removing plots, updating iframe height, minor changes

backend/controller.py

@@ -40,7 +40,7 @@ def interference(
                 raise RuntimeError("There was an error in the selected XAI approach.")
 
         # call the explained chat function
-        prompt_output, history_output, xai_graphic, xai_plot, xai_markup = (
+        prompt_output, history_output, xai_graphic, xai_markup = (
             explained_chat(
                 model=godel,
                 xai=xai,
@@ -61,17 +61,16 @@ def interference(
             knowledge=knowledge,
         )
         # set XAI outputs to disclaimer html/none
-        xai_graphic, xai_plot, xai_markup = (
+        xai_graphic, xai_markup = (
             """
             <div style="text-align: center"><h4>Without Selected XAI Approach,
             no graphic will be displayed</h4></div>
             """,
-            None,
             [("", "")],
         )
 
     # return the outputs
-    return prompt_output, history_output, xai_graphic, xai_plot, xai_markup
+    return prompt_output, history_output, xai_graphic, xai_markup
 
 
 # simple chat function that calls the model
@@ -98,10 +97,10 @@ def explained_chat(
     prompt = model.format_prompt(message, history, system_prompt, knowledge)
 
     # generating an answer using the xai methods explain and respond function
-    answer, xai_graphic, xai_plot, xai_markup = xai.chat_explained(model, prompt)
+    answer, xai_graphic, xai_markup = xai.chat_explained(model, prompt)
 
     # updating the chat history with the new answer
     history.append((message, answer))
 
     # returning the updated history, xai graphic and xai plot elements
-    return "", history, xai_graphic, xai_plot, xai_markup
+    return "", history, xai_graphic, xai_markup

explanation/interpret_shap.py

@@ -26,18 +26,13 @@ def chat_explained(model, prompt):
 
     # create the explanation graphic and plot
     graphic = create_graphic(shap_values)
-    plot = create_plot(
-        values=shap_values.values[0],
-        output_names=shap_values.output_names,
-        input_names=shap_values.data[0],
-    )
     marked_text = markup_text(
         shap_values.data[0], shap_values.values[0], variant="shap"
     )
 
     # create the response text
     response_text = fmt.format_output_text(shap_values.output_names)
-    return response_text, graphic, plot, marked_text
+    return response_text, graphic, marked_text
 
 
 def wrap_shap(model):
@@ -67,55 +62,4 @@ def create_graphic(shap_values):
     graphic_html = plots.text(shap_values, display=False)
 
     # return the html graphic as string
-    return str(graphic_html)
-
-
-# creating an attention heatmap plot using matplotlib/seaborn
-# CREDIT: adopted from official Matplotlib documentation
-## see https://matplotlib.org/stable/
-def create_plot(values, output_names, input_names):
-
-    # Set seaborn style to dark
-    sns.set(style="white")
-    fig, ax = plt.subplots()
-
-    # Setting figure size
-    fig.set_size_inches(
-        max(values.shape[1] * 2, 10),
-        max(values.shape[0] * 1, 5),
-    )
-
-    # Plotting the heatmap with Seaborn's color palette
-    im = ax.imshow(
-        values,
-        vmax=values.max(),
-        vmin=values.min(),
-        cmap=sns.color_palette("vlag_r", as_cmap=True),
-        aspect="auto",
-    )
-
-    # Creating colorbar
-    cbar = ax.figure.colorbar(im, ax=ax)
-    cbar.ax.set_ylabel("Token Attribution", rotation=-90, va="bottom")
-    cbar.ax.yaxis.set_tick_params(color="black")
-    plt.setp(plt.getp(cbar.ax.axes, "yticklabels"), color="black")
-
-    # Setting ticks and labels with white color for visibility
-    ax.set_yticks(np.arange(len(input_names)), labels=input_names)
-    ax.set_xticks(np.arange(len(output_names)), labels=output_names)
-    plt.setp(ax.get_xticklabels(), color="black", rotation=45, ha="right")
-    plt.setp(ax.get_yticklabels(), color="black")
-
-    # Adjusting tick labels
-    ax.tick_params(
-        top=True, bottom=False, labeltop=False, labelbottom=True, color="white"
-    )
-
-    # Adding text annotations with appropriate contrast
-    for i in range(values.shape[0]):
-        for j in range(values.shape[1]):
-            val = values[i, j]
-            color = "white" if im.norm(values.max()) / 2 > im.norm(val) else "black"
-            ax.text(j, i, f"{val:.4f}", ha="center", va="center", color=color)
-
-    return plt
+    return str(graphic_html)

explanation/markup.py

@@ -11,10 +11,13 @@ from utils import formatting as fmt
 def markup_text(input_text: list, text_values: ndarray, variant: str):
     bucket_tags = ["-5", "-4", "-3", "-2", "-1", "0", "+1", "+2", "+3", "+4", "+5"]
 
-    # Flatten the explanations values
+    # Flatten the values depending on the source
+    # attention is averaged, SHAP summed up
     if variant == "shap":
         text_values = np.transpose(text_values)
-    text_values = fmt.flatten_values(text_values)
+        text_values = fmt.flatten_attribution(text_values)
+    else:
+        text_values = fmt.flatten_attention(text_values)
 
     # Determine the minimum and maximum values
     min_val, max_val = np.min(text_values), np.max(text_values)

explanation/visualize.py

@@ -34,74 +34,11 @@ def chat_explained(model, prompt):
         output_attentions=True,
     )
 
-    averaged_attention = avg_attention(attention_output)
+    averaged_attention = fmt.avg_attention(attention_output)
 
-    # create the response text, graphic and plot
+    # create the response text and marked text for ui
     response_text = fmt.format_output_text(decoder_text)
-    plot = create_plot(averaged_attention, (encoder_text, decoder_text))
     marked_text = markup_text(encoder_text, averaged_attention, variant="visualizer")
 
-    return response_text, "", plot, marked_text
+    return response_text, "", marked_text
 
-
-# creating an attention heatmap plot using matplotlib/seaborn
-# CREDIT: adopted from official Matplotlib documentation
-## see https://matplotlib.org/stable/
-def create_plot(averaged_attention_weights, enc_dec_texts: tuple):
-    # transpose the attention weights
-    averaged_attention_weights = np.transpose(averaged_attention_weights)
-
-    # get the encoder and decoder tokens in text form
-    encoder_tokens = enc_dec_texts[0]
-    decoder_tokens = enc_dec_texts[1]
-
-    # set seaborn style to dark and initialize figure and axis
-    sns.set(style="white")
-    fig, ax = plt.subplots()
-
-    # Setting figure size
-    fig.set_size_inches(
-        max(averaged_attention_weights.shape[1] * 2, 10),
-        max(averaged_attention_weights.shape[0] * 1, 5),
-    )
-
-    # Plotting the heatmap with seaborn's color palette
-    im = ax.imshow(
-        averaged_attention_weights,
-        vmax=averaged_attention_weights.max(),
-        vmin=-averaged_attention_weights.min(),
-        cmap=sns.color_palette("rocket", as_cmap=True),
-        aspect="auto",
-    )
-
-    # Creating colorbar
-    cbar = ax.figure.colorbar(im, ax=ax)
-    cbar.ax.set_ylabel("Attention Weight Scale", rotation=-90, va="bottom")
-    cbar.ax.yaxis.set_tick_params(color="black")
-    plt.setp(plt.getp(cbar.ax.axes, "yticklabels"), color="black")
-
-    # Setting ticks and labels with black color for visibility
-    ax.set_yticks(np.arange(len(encoder_tokens)), labels=encoder_tokens)
-    ax.set_xticks(np.arange(len(decoder_tokens)), labels=decoder_tokens)
-    ax.set_title("Attention Weights by Token")
-    plt.setp(ax.get_xticklabels(), color="black", rotation=45, ha="right")
-    plt.setp(ax.get_yticklabels(), color="black")
-
-    # Adding text annotations with appropriate contrast
-    for i in range(averaged_attention_weights.shape[0]):
-        for j in range(averaged_attention_weights.shape[1]):
-            val = averaged_attention_weights[i, j]
-            color = (
-                "white"
-                if im.norm(averaged_attention_weights.max()) / 2 > im.norm(val)
-                else "black"
-            )
-            ax.text(j, i, f"{val:.4f}", ha="center", va="center", color=color)
-
-    # return the plot
-    return plt
-
-
-def avg_attention(attention_values):
-    attention = attention_values.cross_attentions[0][0].detach().numpy()
-    return np.mean(attention, axis=0)

main.py

@@ -180,6 +180,10 @@ with gr.Blocks(
                             " scripts: hieroglyphs, Demotic, and Greek."
                         ),
                     ],
+                    [
+                        "Does money buy happiness?",
+                        ""
+                    ],
                 ],
                 inputs=[user_prompt, knowledge_input],
             )
@@ -197,22 +201,14 @@ with gr.Blocks(
         with gr.Row(variant="panel"):
             # wraps the explanation html to display it statically
             xai_interactive = iFrame(
-                label="Static Explanation",
+                label="Interactive Explanation",
                 value=(
                     '<div style="text-align: center"><h4>No Graphic to Display'
                     " (Yet)</h4></div>"
                 ),
+                height="600px",
                 show_label=True,
             )
-        # row and accordion to display an explanation plot (if applicable)
-        with gr.Row():
-            with gr.Accordion("Token Wise Explanation Plot", open=False):
-                gr.Markdown("""
-                #### Plotted Values
-                Values have been excluded for readability. See colorbar for value indication.
-                """)
-                # plot component that takes a matplotlib figure as input
-                xai_plot = gr.Plot(label="Token Level Explanation")
 
     # functions to trigger the controller
     ## takes information for the chat and the xai selection
@@ -221,13 +217,13 @@ with gr.Blocks(
     submit_btn.click(
         interference,
         [user_prompt, chatbot, knowledge_input, system_prompt, xai_selection],
-        [user_prompt, chatbot, xai_interactive, xai_plot, xai_text],
+        [user_prompt, chatbot, xai_interactive, xai_text],
     )
     # function triggered by the enter key
     user_prompt.submit(
         interference,
         [user_prompt, chatbot, knowledge_input, system_prompt, xai_selection],
-        [user_prompt, chatbot, xai_interactive, xai_plot, xai_text],
+        [user_prompt, chatbot, xai_interactive, xai_text],
     )
 
     # final row to show legal information

public/about.md

@@ -7,7 +7,7 @@ This research tackles the rise of LLM based applications such a chatbots and exp
 ## Links
 
 - [GitHub Repository](https://github.com/LennardZuendorf/thesis-webapp) - The GitHub repository of this project.
-- [HTW Berlin](https://www.htw-berlin.de/) - The University I have built this project for, as part of my thesis.
+- [HTW Berlin](https://www.htw-berlin.de/en/) - The University I have built this project for, as part of my thesis.
 
 
 ## Implementation

utils/formatting.py

@@ -66,5 +66,12 @@ def format_tokens(tokens: list):
 
 
 # function to flatten values into a 2d list by averaging the explanation values
-def flatten_values(values: ndarray, axis: int = 0):
+def flatten_attribution(values: ndarray, axis: int = 0):
+    return np.sum(values, axis=axis)
+
+def flatten_attention(values: ndarray, axis: int = 0):
     return np.mean(values, axis=axis)
+
+def avg_attention(attention_values):
+    attention = attention_values.cross_attentions[0][0].detach().numpy()
+    return np.mean(attention, axis=0)