File size: 1,966 Bytes
6b12e0b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
from predict import run_prediction
from io import StringIO
import json
import spacy
from spacy import displacy
from transformers import pipeline
import torch
import nltk
nltk.download('punkt')
##Summarization
summarizer = pipeline("summarization", model="knkarthick/MEETING_SUMMARY")
def summarize_text(text):
resp = summarizer(text)
stext = resp[0]['summary_text']
return stext
##Company Extraction
ner=pipeline('ner',model='Jean-Baptiste/camembert-ner-with-dates',tokenizer='Jean-Baptiste/camembert-ner-with-dates', aggregation_strategy="simple")
def fin_ner(text):
replaced_spans = ner(text)
new_spans=[]
for item in replaced_spans:
item['entity']=item['entity_group']
del item['entity_group']
new_spans.append(item)
return {"text": text, "entities": new_spans}
#CUAD STARTS
def load_questions():
questions = []
with open('questions.txt') as f:
questions = f.readlines()
return questions
def load_questions_short():
questions_short = []
with open('questionshort.txt') as f:
questions_short = f.readlines()
return questions_short
def quad(query,file):
with open(file) as f:
paragraph = f.read()
questions = load_questions()
questions_short = load_questions_short()
if (not len(paragraph)==0) and not (len(query)==0):
print('getting predictions')
predictions = run_prediction([query], paragraph, 'marshmellow77/roberta-base-cuad',n_best_size=5)
answer = ""
answer_p=""
if predictions['0'] == "":
answer = 'No answer found in document'
else:
with open("nbest.json") as jf:
data = json.load(jf)
for i in range(1):
raw_answer=data['0'][i]['text']
answer += f"{data['0'][i]['text']}\n"
answer_p =answer+ f"Probability: {round(data['0'][i]['probability']*100,1)}%\n\n"
return answer,answer_p
|