update

.env

@@ -1,2 +0,0 @@
-# .env
-PASSWORD=88888888

.gitignore

@@ -1,3 +0,0 @@
- .gitignore
- .env
-test.py

.idea/.gitignore

@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml

.idea/LLM-Open-Generation-Bias.iml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/venv" />
-    </content>
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (LLM-Open-Generation-Bias)" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/LLM-Open-Generation-Bias.iml" filepath="$PROJECT_DIR$/.idea/LLM-Open-Generation-Bias.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

new

@@ -1,30 +1 @@
-newfile_nameerror = "Please enter a valid file name";
-        }
-        else {
-            $newfile_nameerror = "";
-        }
-    }
-    else {
-        $newfile_nameerror = "";
-    }
-
-    if (isset($_POST['newfile_content'])) {
-        $newfile_content = $_POST['newfile_content'];
-        if (empty($newfile_content)) {
-            $newfile_contenterror = "Please enter a valid file content";
-        }
-        else {
-            $newfile_contenterror = "";
-        }
-    }
-    else {
-        $newfile_contenterror = "";
-    }
-
-    if ($newfile_nameerror == "" && $newfile_contenterror == "") {
-        $newfile = fopen($newfile_name, "w");
-        fwrite($newfile, $newfile_content);
-        fclose($newfile);
-        header("Location: index.php");
-    }
-}
+newfile_name

pages/2_new_Demo_1.py

@@ -1,217 +0,0 @@
-import streamlit as st
-import pandas as pd
-from datasets import load_dataset, Dataset
-from random import sample
-from utils.metric import Regard
-from utils.model import gpt2
-import matplotlib.pyplot as plt
-import os
-
-# Set up the Streamlit interface
-st.title('Gender Bias Analysis in Text Generation')
-
-
-def check_password():
-    def password_entered():
-        if password_input == os.getenv('PASSWORD'):
-        # if password_input == "  ":
-            st.session_state['password_correct'] = True
-        else:
-            st.error("Incorrect Password, please try again.")
-
-    password_input = st.text_input("Enter Password:", type="password")
-    submit_button = st.button("Submit", on_click=password_entered)
-
-    if submit_button and not st.session_state.get('password_correct', False):
-        st.error("Please enter a valid password to access the demo.")
-
-
-if not st.session_state.get('password_correct', False):
-    check_password()
-else:
-    st.sidebar.success("Password Verified. Proceed with the demo.")
-
-    if 'data_size' not in st.session_state:
-        st.session_state['data_size'] = 10
-    if 'bold' not in st.session_state:
-        bold = pd.DataFrame({})
-        bold_raw = pd.DataFrame(load_dataset("AlexaAI/bold", split="train"))
-        for index, row in bold_raw.iterrows():
-            bold_raw_prompts = list(row['prompts'])
-            bold_raw_wikipedia = list(row['wikipedia'])
-            bold_expansion = zip(bold_raw_prompts, bold_raw_wikipedia)
-            for bold_prompt, bold_wikipedia in bold_expansion:
-                bold = bold._append(
-                    {'domain': row['domain'], 'name': row['name'], 'category': row['category'], 'prompts': bold_prompt,
-                     'wikipedia': bold_wikipedia}, ignore_index=True)
-        st.session_state['bold'] = Dataset.from_pandas(bold)
-    if 'female_bold' not in st.session_state:
-        st.session_state['female_bold'] = []
-    if 'male_bold' not in st.session_state:
-        st.session_state['male_bold'] = []
-
-    st.subheader('Step 1: Set Data Size')
-    data_size = st.slider('Select number of samples per category:', min_value=1, max_value=50,
-                          value=st.session_state['data_size'])
-    st.session_state['data_size'] = data_size
-
-    if st.button('Show Data'):
-        st.session_state['female_bold'] = sample(
-            [p for p in st.session_state['bold'] if p['category'] == 'American_actresses'], data_size)
-        st.session_state['male_bold'] = sample(
-            [p for p in st.session_state['bold'] if p['category'] == 'American_actors'], data_size)
-
-        st.write(f'Sampled {data_size} female and male American actors.')
-        st.write('**Female Samples:**', pd.DataFrame(st.session_state['female_bold']))
-        st.write('**Male Samples:**', pd.DataFrame(st.session_state['male_bold']))
-
-    if st.session_state['female_bold'] and st.session_state['male_bold']:
-        st.subheader('Step 2: Generate Text')
-
-        if st.button('Generate Text'):
-            GPT2 = gpt2()
-            st.session_state['male_prompts'] = [p['prompts'] for p in st.session_state['male_bold']]
-            st.session_state['female_prompts'] = [p['prompts'] for p in st.session_state['female_bold']]
-            st.session_state['male_wiki_continuation'] = [p['wikipedia'].replace(p['prompts'], '') for p in
-                                                          st.session_state['male_bold']]
-            st.session_state['female_wiki_continuation'] = [p['wikipedia'].replace(p['prompts'], '') for p in
-                                                            st.session_state['female_bold']]
-
-            progress_bar = st.progress(0)
-
-            st.write('Generating text for male prompts...')
-            male_generation = GPT2.text_generation(st.session_state['male_prompts'], pad_token_id=50256, max_length=50,
-                                                   do_sample=False, truncation=True)
-            st.session_state['male_continuations'] = [gen[0]['generated_text'].replace(prompt, '') for gen, prompt in
-                                                      zip(male_generation, st.session_state['male_prompts'])]
-
-            progress_bar.progress(50)
-
-            st.write('Generating text for female prompts...')
-            female_generation = GPT2.text_generation(st.session_state['female_prompts'], pad_token_id=50256,
-                                                     max_length=50, do_sample=False, truncation=True)
-            st.session_state['female_continuations'] = [gen[0]['generated_text'].replace(prompt, '') for gen, prompt in
-                                                        zip(female_generation, st.session_state['female_prompts'])]
-
-            progress_bar.progress(100)
-            st.write('Text generation completed.')
-
-    if st.session_state.get('male_continuations') and st.session_state.get('female_continuations'):
-        st.subheader('Step 3: Sample Generated Texts')
-
-        st.write("Male Data Samples:")
-        samples_df = pd.DataFrame({
-            'Male Prompt': st.session_state['male_prompts'],
-            'Male Continuation': st.session_state['male_continuations'],
-            'Male Wiki Continuation': st.session_state['male_wiki_continuation'],
-        })
-        st.write(samples_df)
-
-        st.write("Female Data Samples:")
-        samples_df = pd.DataFrame({
-            'Female Prompt': st.session_state['female_prompts'],
-            'Female Continuation': st.session_state['female_continuations'],
-            'Female Wiki Continuation': st.session_state['female_wiki_continuation'],
-        })
-        st.write(samples_df)
-
-        if st.button('Evaluate'):
-            st.subheader('Step 4: Regard Results')
-            regard = Regard("inner_compare")
-            st.write('Computing regard results to compare male and female continuations...')
-
-            with st.spinner('Computing regard results...'):
-                regard_male_results = regard.compute(data=st.session_state['male_continuations'],
-                                                     references=st.session_state['male_wiki_continuation'])
-                st.write('**Raw Regard Results:**')
-                st.json(regard_male_results)
-                st.session_state['rmr'] = regard_male_results
-
-                regard_female_results = regard.compute(data=st.session_state['female_continuations'],
-                                                       references=st.session_state['female_wiki_continuation'])
-                st.write('**Average Regard Results:**')
-                st.json(regard_female_results)
-                st.session_state['rfr'] = regard_female_results
-
-        if st.button('Plot'):
-            st.subheader('Step 5: Regard Results Plotting')
-            categories = ['GPT2', 'Wiki']
-
-            mp_gpt = st.session_state['rmr']['no_ref_diff_mean']['positive']
-            mn_gpt = st.session_state['rmr']['no_ref_diff_mean']['negative']
-            mo_gpt = 1 - (mp_gpt + mn_gpt)
-
-            mp_wiki = mp_gpt - st.session_state['rmr']['ref_diff_mean']['positive']
-            mn_wiki = mn_gpt -st.session_state['rmr']['ref_diff_mean']['negative']
-            mo_wiki = 1 - (mn_wiki + mp_wiki)
-
-            fp_gpt = st.session_state['rfr']['no_ref_diff_mean']['positive']
-            fn_gpt = st.session_state['rfr']['no_ref_diff_mean']['negative']
-            fo_gpt = 1 - (fp_gpt + fn_gpt)
-
-            fp_wiki = fp_gpt - st.session_state['rfr']['ref_diff_mean']['positive']
-            fn_wiki = fn_gpt - st.session_state['rfr']['ref_diff_mean']['negative']
-            fo_wiki = 1 - (fn_wiki + fp_wiki)
-
-            positive_m = [mp_gpt, mp_wiki]
-            other_m = [mo_gpt, mo_wiki]
-            negative_m = [mn_gpt, mn_wiki]
-
-            positive_f = [fp_gpt, fp_wiki]
-            other_f = [fo_gpt, fo_wiki]
-            negative_f = [fn_gpt, fn_wiki]
-
-            # Plotting
-            fig_a, ax_a = plt.subplots()
-            ax_a.bar(categories, negative_m, label='Negative', color='blue')
-            ax_a.bar(categories, other_m, bottom=negative_m, label='Other', color='orange')
-            ax_a.bar(categories, positive_m, bottom=[negative_m[i] + other_m[i] for i in range(len(negative_m))],
-                     label='Positive', color='green')
-
-            plt.xlabel('Categories')
-            plt.ylabel('Proportion')
-            plt.title('GPT vs Wiki on male regard')
-            plt.legend()
-
-            st.pyplot(fig_a)
-
-            fig_b, ax_b = plt.subplots()
-            ax_b.bar(categories, negative_f, label='Negative', color='blue')
-            ax_b.bar(categories, other_f, bottom=negative_f, label='Other', color='orange')
-            ax_b.bar(categories, positive_f, bottom=[negative_f[i] + other_f[i] for i in range(len(negative_f))],
-                     label='Positive', color='green')
-
-            plt.xlabel('Categories')
-            plt.ylabel('Proportion')
-            plt.title('GPT vs Wiki on female regard')
-            plt.legend()
-            st.pyplot(fig_b)
-
-            m_increase = mp_gpt - mn_gpt
-            m_relative_increase = mp_gpt - mp_wiki - (mn_gpt - mn_wiki)
-            f_increase = fp_gpt - fn_gpt
-            f_relative_increase = fp_gpt - fp_wiki - (fn_gpt - fn_wiki)
-
-            absolute_difference = [m_increase, f_increase]
-            relative_difference = [m_relative_increase, f_relative_increase]
-
-            new_categories = ['Male', 'Female']
-
-            fig_c, ax_c = plt.subplots()
-            ax_c.bar(new_categories, absolute_difference, label='Positive - Negative', color='#40E0D0')
-
-            plt.xlabel('Categories')
-            plt.ylabel('Proportion')
-            plt.title('Difference of positive and negative: Male vs Female')
-            plt.legend()
-            st.pyplot(fig_c)
-
-            fig_d, ax_d = plt.subplots()
-            ax_d.bar(new_categories, relative_difference, label='Positive - Negative', color='#40E0D0')
-
-            plt.xlabel('Categories')
-            plt.ylabel('Proportion')
-            plt.title('Difference of positive and negative (relative to Wiki): Male vs Female')
-            plt.legend()
-            st.pyplot(fig_d)
-

requirements.txt

@@ -1,4 +1,3 @@
 openai
 transformers
-torch==2.0.1
-matplotlib
+torch==2.0.1

utils/metric.py

@@ -43,27 +43,13 @@ class Regard:
                 return {"average_data_regard": pred_mean, "average_references_regard": ref_mean}
             else:
                 return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}}
-        elif self.config_name == "inner_compare":
+        else:
             pred_scores, pred_regard = self.regard(data)
-            ref_scores, ref_regard = self.regard(references)
-
-            postive_pred_regard = pred_regard['positive']
-            positive_ref_regard = ref_regard['positive']
-            postive_diff_regard = list(range(len(postive_pred_regard)))
-            for score_index in range(len(postive_pred_regard)):
-                postive_diff_regard[score_index] = postive_pred_regard[score_index] - positive_ref_regard[score_index]
-
-            negative_pred_regard = pred_regard['negative']
-            negative_ref_regard = ref_regard['negative']
-            negative_diff_regard = list(range(len(negative_pred_regard)))
-            for score_index in range(len(negative_pred_regard)):
-                negative_diff_regard[score_index] = negative_pred_regard[score_index] - negative_ref_regard[score_index]
-
-            ref_diff_regard = {'positive': postive_diff_regard, 'negative': negative_diff_regard}
-            ref_diff_mean = {k: mean(v) for k, v in ref_diff_regard.items()}
-            no_ref_diff_regard = {'positive': postive_pred_regard, 'negative': negative_pred_regard}
-            no_ref_diff_mean = {k: mean(v) for k, v in no_ref_diff_regard.items()}
-
-            return {"ref_diff_mean": ref_diff_mean,
-                    'no_ref_diff_mean': no_ref_diff_mean}
-
+            pred_mean = {k: mean(v) for k, v in pred_regard.items()}
+            pred_max = {k: max(v) for k, v in pred_regard.items()}
+            if aggregation == "maximum":
+                return {"max_regard": pred_max}
+            elif aggregation == "average":
+                return {"average_regard": pred_mean}
+            else:
+                return {"regard": pred_scores}