|
import streamlit as st |
|
from predict import run_prediction |
|
from io import StringIO |
|
import json |
|
import spacy |
|
from spacy import displacy |
|
from transformers import AutoTokenizer, AutoModelForTokenClassification,RobertaTokenizer,pipeline |
|
import torch |
|
import nltk |
|
from nltk.tokenize import sent_tokenize |
|
from fin_readability_sustainability import BERTClass, do_predict |
|
import pandas as pd |
|
import en_core_web_sm |
|
nlp = en_core_web_sm.load() |
|
nltk.download('punkt') |
|
|
|
|
|
st.set_page_config(layout="wide") |
|
st.cache(show_spinner=False, persist=True) |
|
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") |
|
|
|
tokenizer_sus = RobertaTokenizer.from_pretrained('roberta-base') |
|
model_sustain = BERTClass(2, "sustanability") |
|
model_sustain.to(device) |
|
model_sustain.load_state_dict(torch.load('sustainability_model.bin', map_location=device)['model_state_dict']) |
|
|
|
|
|
def get_sustainability(text): |
|
df = pd.DataFrame({'sentence':sent_tokenize(text)}) |
|
actual_predictions_sustainability = do_predict(model_sustain, tokenizer_sus, df) |
|
highlight = [] |
|
for sent, prob in zip(df['sentence'].values, actual_predictions_sustainability[1]): |
|
if prob>=4.384316: |
|
highlight.append((sent, 'non-sustainable')) |
|
elif prob<=1.423736: |
|
highlight.append((sent, 'sustainable')) |
|
else: |
|
highlight.append((sent, '-')) |
|
return highlight |
|
|
|
|
|
|
|
|
|
def summarize_text(text): |
|
summarizer = pipeline("summarization", model="knkarthick/MEETING_SUMMARY") |
|
resp = summarizer(text) |
|
stext = resp[0]['summary_text'] |
|
return stext |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_questions(): |
|
questions = [] |
|
with open('questions.txt') as f: |
|
questions = f.readlines() |
|
return questions |
|
|
|
|
|
def load_questions_short(): |
|
questions_short = [] |
|
with open('questionshort.txt') as f: |
|
questions_short = f.readlines() |
|
return questions_short |
|
|
|
|
|
st.cache(show_spinner=False, persist=True) |
|
|
|
|
|
questions = load_questions() |
|
questions_short = load_questions_short() |
|
|
|
|
|
st.sidebar.title("Interactive Contract Analysis") |
|
|
|
st.sidebar.header('CONTRACT UPLOAD') |
|
|
|
with open('NDA1.txt') as f: |
|
contract_data = f.read() |
|
|
|
user_upload = st.sidebar.file_uploader('Please upload your contract', type=['txt'], |
|
accept_multiple_files=False) |
|
|
|
|
|
|
|
if user_upload is not None: |
|
print(user_upload.name, user_upload.type) |
|
extension = user_upload.name.split('.')[-1].lower() |
|
if extension == 'txt': |
|
print('text file uploaded') |
|
|
|
stringio = StringIO(user_upload.getvalue().decode("utf-8")) |
|
|
|
|
|
contract_data = stringio.read() |
|
else: |
|
st.warning('Unknown uploaded file type, please try again') |
|
|
|
results_drop = ['1', '2', '3'] |
|
number_results = st.sidebar.selectbox('Select number of results', results_drop) |
|
|
|
|
|
st.header("Legal Contract Review Demo") |
|
paragraph = st.text_area(label="Contract", value=contract_data, height=300) |
|
|
|
questions_drop = questions_short |
|
question_short = st.selectbox('Choose one of the 41 queries from the CUAD dataset:', questions_drop) |
|
idxq = questions_drop.index(question_short) |
|
question = questions[idxq] |
|
|
|
|
|
raw_answer="" |
|
if st.button('Analyze'): |
|
if (not len(paragraph)==0) and not (len(question)==0): |
|
print('getting predictions') |
|
with st.spinner(text='Analysis in progress...'): |
|
predictions = run_prediction([question], paragraph, 'marshmellow77/roberta-base-cuad', |
|
n_best_size=5) |
|
answer = "" |
|
if predictions['0'] == "": |
|
answer = 'No answer found in document' |
|
else: |
|
|
|
|
|
|
|
|
|
answer = "" |
|
with open("nbest.json") as jf: |
|
data = json.load(jf) |
|
for i in range(int(number_results)): |
|
raw_answer=data['0'][i]['text'] |
|
answer += f"Answer {i+1}: {data['0'][i]['text']} -- \n" |
|
answer += f"Probability: {round(data['0'][i]['probability']*100,1)}%\n\n" |
|
st.success(answer) |
|
st.write(get_sustainability(raw_answer)) |
|
st.write(summarize_text(raw_answer)) |
|
doc = nlp(raw_answer) |
|
st.write(displacy.render(doc, style="ent")) |
|
else: |
|
st.write("Unable to call model, please select question and contract") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|