import gradio as gr
from transformers import AutoTokenizer, AutoModel
from sentence_transformers import SentenceTransformer
import pickle
import nltk
import time
from input_format import *
from score import *
nltk.download('punkt') # tokenizer
nltk.download('averaged_perceptron_tagger') # postagger
## load document scoring model
#torch.cuda.is_available = lambda : False # uncomment to test with CPU only
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#pretrained_model = 'allenai/specter'
pretrained_model = 'allenai/specter2'
tokenizer = AutoTokenizer.from_pretrained(pretrained_model)
doc_model = AutoModel.from_pretrained(pretrained_model)
doc_model.to(device)
## load sentence model
sent_model = doc_model # have the same model for document and sentence level
# OR specify different model for sentence level
#sent_model = SentenceTransformer('sentence-transformers/gtr-t5-base')
#sent_model.to(device)
NUM_PAPERS_SHOW = 5 # max number of top papers to show from the reviewer upfront
NUM_PAIRS_SHOW = 5 # max number of top sentence pairs to show
def get_similar_paper(
title_input,
abstract_text_input,
author_id_input,
top_paper_slider,
top_pair_slider,
results={}, # this state variable will be updated and returned
):
progress = gr.Progress()
if title_input == None:
title_input = '' # if no title is given, just focus on abstract.
print('retrieving similar papers...')
start = time.time()
input_sentences = sent_tokenize(abstract_text_input)
# Get author papers from id
#progress(0.1, desc="Retrieving reviewer papers ...")
name, papers = get_text_from_author_id(author_id_input)
# Compute Doc-level affinity scores for the Papers
# print('computing document scores...')
#progress(0.5, desc="Computing document scores...")
# TODO detect duplicate papers?
titles, abstracts, paper_urls, doc_scores = compute_document_score(
doc_model,
tokenizer,
title_input,
abstract_text_input,
papers,
batch=10
)
results = {
'name': name,
'author_url': author_id_input,
'titles': titles,
'abstracts': abstracts,
'urls': paper_urls,
'doc_scores': doc_scores
}
# Select top 10 papers to show
titles = titles[:10]
abstracts = abstracts[:10]
doc_scores = doc_scores[:10]
paper_urls = paper_urls[:10]
display_title = ['[ %0.3f ] %s'%(s, t) for t, s in zip(titles, doc_scores)]
end = time.time()
retrieval_time = end - start
print('paper retrieval complete in [%0.2f] seconds'%(retrieval_time))
progress(0.9, desc="Obtaining relevant information from the papers...")
print('obtaining highlights..')
start = time.time()
input_sentences = sent_tokenize(abstract_text_input)
num_input_sents = len(input_sentences)
for aa, (tt, ab, ds, url) in enumerate(zip(titles, abstracts, doc_scores, paper_urls)):
# Compute sent-level and phrase-level affinity scores for each papers
sent_ids, sent_scores, info, top_pairs_info = get_highlight_info(
sent_model,
tokenizer,
abstract_text_input,
ab,
K=None,
top_pair_num=10, # top ten sentence pairs at max to show upfront
)
num_cand_sents = sent_ids.shape[1]
# get scores for each word in the format for Gradio Interpretation component
word_scores = dict()
for i in range(num_input_sents):
word_scores[str(i)] = dict()
for j in range(1, num_cand_sents+1):
ww, ss = remove_spaces(info['all_words'], info[i][j]['scores'])
word_scores[str(i)][str(j)] = {
"original": ab,
"interpretation": list(zip(ww, ss))
}
results[display_title[aa]] = {
'title': tt,
'abstract': ab,
'num_cand_sents': num_cand_sents,
'doc_score': '%0.3f'%ds,
'source_sentences': input_sentences,
'highlight': word_scores,
'top_pairs': top_pairs_info,
'url': url
}
end = time.time()
highlight_time = end - start
print('done in [%0.2f] seconds'%(highlight_time))
## Set up output elements
## Components for Initial Part
result1_desc_value = """
For each paper, top %d sentence pairs (one from the submission, one from the paper) with the highest relevance scores are shown.
**Blue highlights**: phrases that appear in both sentences.
"""%(int(top_paper_slider), author_id_input, results['name'], int(top_pair_slider))
out1 = [
gr.update(visible=True), # Explore more button
gr.update(value=result1_desc_value, visible=True), # result 1 description
gr.update(value='Done (in %0.1f seconds)'%(retrieval_time+highlight_time), visible=True), # search status
gr.update(visible=True), # top paper slider
gr.update(visible=True) # top pair slider
]
### Components for Results in Initial Part
top_papers_show = int(top_paper_slider) # number of top papers to show upfront
top_num_info_show = int(top_pair_slider) # number of sentence pairs from each paper to show upfront
output = setup_outputs(results, top_papers_show, top_num_info_show)
out2 = []
for x in output:
out2 += x
### Components for Explore More Section
# list of top papers, sentences to select from, paper_title, affinity
title = results[display_title[0]]['title'] # set default title as the top paper
url = results[display_title[0]]['url']
aff_score = results[display_title[0]]['doc_score']
title_out = """
"""%(url, title)
aff_score_out = '##### Affinity Score: %s'%aff_score
result2_desc_value = """
##### Click a paper by %s (left, sorted by affinity scores), and a sentence from the submission (center), to see which parts of the paper are relevant (right).
"""%results['name']
out3 = [
gr.update(choices=display_title, value=display_title[0], interactive=True), # set of papers (radio)
gr.update(choices=input_sentences, value=input_sentences[0], interactive=True), # submission sentences
gr.update(value=title_out), # paper_title
gr.update(value=aff_score_out), # affinity
gr.update(value=result2_desc_value), # result 2 description (show more section)
gr.update(value=1, maximum=len(sent_tokenize(abstracts[0]))), # highlight slider to control
]
## Return by adding the State variable info
return out1 + out2 + out3 + [results]
def setup_outputs(info, top_papers_show, top_num_info_show):
titles = info['titles']
doc_scores = info['doc_scores']
paper_urls = info['urls']
display_title = ['[ %0.3f ] %s'%(s, t) for t, s in zip(info['titles'], info['doc_scores'])]
title = []
affinity = []
sent_pair_score = []
sent_text_query = []
sent_text_candidate = []
sent_hl_query = []
sent_hl_candidate = []
demarc_lines = []
for i in range(top_papers_show):
if i == 0:
title.append(
gr.update(value="""
"""%(paper_urls[i], titles[i]), visible=True)
)
affinity.append(
gr.update(value='#### Affinity Score: %0.3f'%doc_scores[i], visible=True) # document affinity
)
demarc_lines.append(gr.Markdown.update(visible=True))
# fill in the rest as
tp = info[display_title[i]]['top_pairs']
for j in range(top_num_info_show):
if i == 0 and j == 0:
# for the first entry add help tip
sent_pair_score.append(
gr.update(value="""Sentence Relevance:\n%0.3f
Measures how similar the sentence pairs are.
"""%tp[j]['score'], visible=True)
)
else:
sent_pair_score.append(
gr.Textbox.update(value='Sentence Relevance:\n%0.3f'%tp[j]['score'], visible=True)
)
sent_text_query.append(gr.Textbox.update(tp[j]['query']['original']))
sent_text_candidate.append(gr.Textbox.update(tp[j]['candidate']['original']))
sent_hl_query.append(tp[j]['query'])
sent_hl_candidate.append(tp[j]['candidate'])
#row2.append(gr.update(visible=True))
sent_pair_score += [gr.Markdown.update(visible=False)] * (NUM_PAIRS_SHOW - top_num_info_show)
sent_text_query += [gr.Textbox.update(value='', visible=False)] * (NUM_PAIRS_SHOW - top_num_info_show)
sent_text_candidate += [gr.Textbox.update(value='', visible=False)] * (NUM_PAIRS_SHOW - top_num_info_show)
sent_hl_query += [None] * (NUM_PAIRS_SHOW - top_num_info_show)
sent_hl_candidate += [None] * (NUM_PAIRS_SHOW - top_num_info_show)
# mark others not visible
title += [gr.Markdown.update(visible=False)] * (NUM_PAPERS_SHOW - top_papers_show)
affinity += [gr.Markdown.update(visible=False)] * (NUM_PAPERS_SHOW - top_papers_show)
demarc_lines += [gr.Markdown.update(visible=False)] * (NUM_PAPERS_SHOW - top_papers_show)
sent_pair_score += [gr.Markdown.update(visible=False)] * (NUM_PAPERS_SHOW - top_papers_show) * NUM_PAIRS_SHOW
sent_text_query += [gr.Textbox.update(value='', visible=False)] * (NUM_PAPERS_SHOW - top_papers_show) * NUM_PAIRS_SHOW
sent_text_candidate += [gr.Textbox.update(value='', visible=False)] * (NUM_PAPERS_SHOW - top_papers_show) * NUM_PAIRS_SHOW
sent_hl_query += [None] * (NUM_PAPERS_SHOW - top_papers_show) * NUM_PAIRS_SHOW
sent_hl_candidate += [None] * (NUM_PAPERS_SHOW - top_papers_show) * NUM_PAIRS_SHOW
assert(len(title) == NUM_PAPERS_SHOW)
assert(len(affinity) == NUM_PAPERS_SHOW)
assert(len(sent_pair_score) == NUM_PAIRS_SHOW * NUM_PAPERS_SHOW)
return title, affinity, demarc_lines, sent_pair_score, sent_text_query, sent_text_candidate, sent_hl_query, sent_hl_candidate
def show_more(info):
# show the interactive part of the app
return (
gr.update(visible=True), # description
gr.update(visible=True), # set of papers
gr.update(visible=True), # submission sentences
gr.update(visible=True), # title row
gr.update(visible=True), # affinity row
gr.update(visible=True), # highlight legend
gr.update(visible=True), # highlight slider
gr.update(visible=True), # highlight abstract
)
def show_status():
# show search status field when search button is clicked
return gr.update(visible=True)
def update_name(author_id_input):
# update the name of the author based on the id input
name, _ = get_text_from_author_id(author_id_input)
return gr.update(value=name)
def change_sentence(
selected_papers_radio,
source_sent_choice,
highlight_slider,
info={}
):
# change the output highlight based on the sentence selected from the submission
if len(info.keys()) != 0: # if the info is not empty
source_sents = info[selected_papers_radio]['source_sentences']
highlights = info[selected_papers_radio]['highlight']
idx = source_sents.index(source_sent_choice)
return highlights[str(idx)][str(highlight_slider)]
else:
return
def change_paper(
selected_papers_radio,
source_sent_choice,
highlight_slider,
info={}
):
if len(info.keys()) != 0: # if the info is not empty
source_sents = info[selected_papers_radio]['source_sentences']
title = info[selected_papers_radio]['title']
num_sents = info[selected_papers_radio]['num_cand_sents']
abstract = info[selected_papers_radio]['abstract']
aff_score = info[selected_papers_radio]['doc_score']
highlights = info[selected_papers_radio]['highlight']
url = info[selected_papers_radio]['url']
title_out = """
"""%(url, title)
aff_score_out = '##### Affinity Score: %s'%aff_score
idx = source_sents.index(source_sent_choice)
if highlight_slider <= num_sents:
return title_out, abstract, aff_score_out, highlights[str(idx)][str(highlight_slider)], gr.update(value=highlight_slider, maximum=num_sents)
else: # if the slider is set to more than the current number of sentences, show the max number of highlights
return title_out, abstract, aff_score_out, highlights[str(idx)][str(num_sents)], gr.update(value=num_sents, maximum=num_sents)
else:
return
def change_num_highlight(
selected_papers_radio,
source_sent_choice,
highlight_slider,
info={}
):
if len(info.keys()) != 0: # if the info is not empty
source_sents = info[selected_papers_radio]['source_sentences']
highlights = info[selected_papers_radio]['highlight']
idx = source_sents.index(source_sent_choice)
return highlights[str(idx)][str(highlight_slider)]
else:
return
def change_top_output(top_paper_slider, top_pair_slider, info={}):
top_papers_show = int(top_paper_slider)
top_num_info_show = int(top_pair_slider)
result1_desc_value = """
For each paper, top %d sentence pairs (one from the submission, one from the paper) with the highest relevance scores are shown.
**Blue highlights**: phrases that appear in both sentences.
"""%(int(top_paper_slider), info['author_url'], info['name'], int(top_pair_slider))
if len(info.keys()) != 0:
tmp = setup_outputs(info, top_papers_show, top_num_info_show)
x = []
for t in tmp:
x += t
return x + [gr.update(value=result1_desc_value)]
else:
return
def reinit_hl(top_paper_slider, top_pair_slider, *args):
args = list(args)
base = 3*NUM_PAPERS_SHOW+NUM_PAPERS_SHOW*NUM_PAIRS_SHOW
increment = NUM_PAPERS_SHOW*NUM_PAIRS_SHOW
text_query = args[base:base+increment]
text_candidate = args[base+increment:base+2*increment]
hl_query = args[base+2*increment:base+3*increment]
hl_candidate = args[base+3*increment:base+4*increment]
for i in range(int(top_paper_slider)):
for j in range(int(top_pair_slider),NUM_PAIRS_SHOW):
hl_query[i*NUM_PAIRS_SHOW+j] = gr.components.Interpretation(text_query[i*NUM_PAIRS_SHOW+j])
hl_candidate[i*NUM_PAIRS_SHOW+j] = gr.components.Interpretation(text_candidate[i*NUM_PAIRS_SHOW+j])
for i in range(int(top_paper_slider),NUM_PAPERS_SHOW):
for j in range(NUM_PAPERS_SHOW):
hl_query[i*NUM_PAIRS_SHOW+j] = gr.components.Interpretation(text_query[i*NUM_PAIRS_SHOW+j])
hl_candidate[i*NUM_PAIRS_SHOW+j] = gr.components.Interpretation(text_candidate[i*NUM_PAIRS_SHOW+j])
args[base:base+increment] = text_query
args[base+increment:base+2*increment] = text_candidate
args[base+2*increment:base+3*increment] = hl_query
args[base+3*increment:base+4*increment] = hl_candidate
return args
with gr.Blocks(css='style.css') as demo:
info = gr.State({}) # cached search results as a State variable shared throughout
# Text description about the app and disclaimer
### TEXT Description
# General instruction
general_instruction = """
# R2P2: An Assistance Tool for Reviewer-Paper Matching in Peer Review
#### Who is it for?
It is for meta-reviewers, area chairs, program chairs, or anyone who oversees the submission-reviewer matching process in peer review for academic conferences, journals, and grants.
#### How does it help?
A typical meta-reviewer workflow lacks supportive information on **what makes the pre-selected candidate reviewers a good fit** for the submission. Only affinity scores between the reviewer and the paper are shown, without additional detail.
R2P2 provides more information about each reviewer. It searches for the **most relevant papers** among the reviewer's previous publications and **highlights relevant parts** within them.
"""
# More details (video, addendum)
more_details_instruction = """Check out this video for a quick introduction of what R2P2 is and how it can help. You can find more details here, along with our privacy policy and disclaimer."""
gr.Markdown(general_instruction)
gr.HTML(more_details_instruction)
gr.Markdown("""---""")
### INPUT
with gr.Row() as input_row:
with gr.Column(scale=3):
with gr.Row():
title_input = gr.Textbox(label='Submission Title', info='Paste in the title of the submission.')
with gr.Row():
abstract_text_input = gr.Textbox(label='Submission Abstract', info='Paste in the abstract of the submission.')
with gr.Column(scale=2):
with gr.Row():
author_id_input = gr.Textbox(label='Reviewer Profile Link (Semantic Scholar)', info="Paste in the reviewer's Semantic Scholar link")
with gr.Row():
name = gr.Textbox(label='Confirm Reviewer Name', info='This will be automatically updated based on the reviewer profile link above', interactive=False)
author_id_input.change(fn=update_name, inputs=author_id_input, outputs=name)
# Add examples
example_title ="The Toronto Paper Matching System: An automated paper-reviewer assignment system"
example_submission = """One of the most important tasks of conference organizers is the assignment of papers to reviewers. Reviewers' assessments of papers is a crucial step in determining the conference program, and in a certain sense to shape the direction of a field. However this is not a simple task: large conferences typically have to assign hundreds of papers to hundreds of reviewers, and time constraints make the task impossible for one person to accomplish. Furthermore other constraints, such as reviewer load have to be taken into account, preventing the process from being completely distributed. We built the first version of a system to suggest reviewer assignments for the NIPS 2010 conference, followed, in 2012, by a release that better integrated our system with Microsoft's popular Conference Management Toolkit (CMT). Since then our system has been widely adopted by the leading conferences in both the machine learning and computer vision communities. This paper provides an overview of the system, a summary of learning models and methods of evaluation that we have been using, as well as some of the recent progress and open issues."""
example_reviewer = "https://www.semanticscholar.org/author/Nihar-B.-Shah/1737249"
gr.Examples(
examples=[[example_title, example_submission, example_reviewer]],
inputs=[title_input, abstract_text_input, author_id_input],
cache_examples=False,
label="Try out the following example input."
)
with gr.Row():
compute_btn = gr.Button('What Makes This a Good Match?')
with gr.Row():
search_status = gr.Textbox(label='Search Status', interactive=False, visible=False)
### OVERVIEW RESULTS
# Paper title, score, and top-ranking sentence pairs
# a knob for controlling the number of output displayed
with gr.Row():
with gr.Column(scale=5):
result1_desc = gr.Markdown(value='', visible=False)
with gr.Column(scale=2):
with gr.Row():
top_paper_slider = gr.Slider(label='Top-K Papers by the Reviewer', value=3, minimum=3, step=1, maximum=NUM_PAPERS_SHOW, visible=False)
with gr.Row():
top_pair_slider = gr.Slider(label='Top-K Sentence Pairs per Paper', value=2, minimum=2, step=1, maximum=NUM_PAIRS_SHOW, visible=False)
paper_title_up = []
paper_affinity_up = []
sent_pair_score = []
sent_text_query = []
sent_text_candidate = []
sent_hl_query = []
sent_hl_candidate = []
demarc_lines = []
row_elems1 = []
row_elems2 = []
for i in range(NUM_PAPERS_SHOW):
with gr.Row():
with gr.Column(scale=3):
tt = gr.Markdown(value='', visible=False)
paper_title_up.append(tt)
with gr.Column(scale=1):
aff = gr.Markdown(value='', visible=False)
paper_affinity_up.append(aff)
for j in range(NUM_PAIRS_SHOW):
with gr.Row():
with gr.Column(scale=1):
sps = gr.Markdown(value='', visible=False)
sent_pair_score.append(sps)
with gr.Column(scale=5):
stq = gr.Textbox(label='Sentence from Submission', visible=False)
shq = gr.components.Interpretation(stq, visible=False)
sent_text_query.append(stq)
sent_hl_query.append(shq)
with gr.Column(scale=5):
stc = gr.Textbox(label="Sentence from Reviewer's Paper", visible=False)
shc = gr.components.Interpretation(stc, visible=False)
sent_text_candidate.append(stc)
sent_hl_candidate.append(shc)
with gr.Row():
dml = gr.Markdown("""---""", visible=False)
demarc_lines.append(dml)
## Show more button
with gr.Row():
see_more_rel_btn = gr.Button('Explore more', visible=False)
### PAPER INFORMATION
# Description for Explore More Section
with gr.Row():
result2_desc = gr.Markdown(value='', visible=False)
# Highlight description
hl_desc = """
**Red**: sentences simiar to the selected sentence from submission. Darker = more similar.
**Blue**: phrases that appear in both sentences.
"""
#---"""
# show multiple papers in radio check box to select from
paper_abstract = gr.Textbox(label='Abstract', interactive=False, visible=False)
with gr.Row():
with gr.Column(scale=1):
selected_papers_radio = gr.Radio(
choices=[], # will be udpated with the button click
visible=False, # also will be updated with the button click
label='Top Relevant Papers from the Reviewer'
)
with gr.Column(scale=2):
# sentences from submission
source_sentences = gr.Radio(
choices=[],
visible=False,
label='Sentences from Submission Abstract',
)
with gr.Column(scale=3):
# selected paper and highlight
with gr.Row():
# slider for highlight amount
highlight_slider = gr.Slider(
label='Number of Highlighted Sentences',
minimum=1,
maximum=15,
step=1,
value=2,
visible=False
)
with gr.Row():
# highlight legend
highlight_legend = gr.Markdown(value=hl_desc, visible=False)
with gr.Row(visible=False) as title_row:
# selected paper title
paper_title = gr.Markdown(value='')
with gr.Row(visible=False) as aff_row:
# selected paper's affinity score
affinity = gr.Markdown(value='')
with gr.Row(visible=False) as hl_row:
# highlighted text from paper
highlight = gr.components.Interpretation(paper_abstract)
### EVENT LISTENERS
# components to work with
init_components = [
see_more_rel_btn, # explore more button
result1_desc, # description for first results
search_status, # search status
top_paper_slider,
top_pair_slider
]
init_result_components = \
paper_title_up + paper_affinity_up + demarc_lines + sent_pair_score + \
sent_text_query + sent_text_candidate + sent_hl_query + sent_hl_candidate
explore_more_components = [
selected_papers_radio, # list of papers for show more section
source_sentences, # list of sentences for show more section
paper_title, # paper title for show more section
affinity, # affinity for show more section
result2_desc, # description for explore more
highlight_slider, # highlight slider
]
compute_btn.click(
fn=show_status,
inputs=[],
outputs=search_status
)
compute_btn.click(
fn=get_similar_paper,
inputs=[
title_input,
abstract_text_input,
author_id_input,
top_paper_slider,
top_pair_slider,
info
],
outputs=init_components + init_result_components + explore_more_components + [info],
show_progress=True,
scroll_to_output=True
)
# Get more info (move to more interactive portion)
see_more_rel_btn.click(
fn=show_more,
inputs=info,
outputs=[
result2_desc,
selected_papers_radio,
source_sentences,
title_row,
aff_row,
highlight_legend,
highlight_slider,
hl_row,
]
)
# change highlight based on selected sentences from submission
source_sentences.change(
fn=change_sentence,
inputs=[
selected_papers_radio,
source_sentences,
highlight_slider,
info
],
outputs=highlight
)
# change paper to show based on selected papers
selected_papers_radio.change(
fn=change_paper,
inputs=[
selected_papers_radio,
source_sentences,
highlight_slider,
info,
],
outputs= [
paper_title,
paper_abstract,
affinity,
highlight,
highlight_slider
]
)
# change number of higlights to show
highlight_slider.change(
fn=change_num_highlight,
inputs=[
selected_papers_radio,
source_sentences,
highlight_slider,
info
],
outputs=[
highlight
]
)
# change number of top papers to show initially
top_paper_slider.change(
fn=change_top_output,
inputs=[
top_paper_slider,
top_pair_slider,
info
],
outputs=init_result_components+[result1_desc]
)
# change number of top sentence pairs to show initially
top_pair_slider.change(
fn=change_top_output,
inputs=[
top_paper_slider,
top_pair_slider,
info
],
outputs=init_result_components+[result1_desc]
)
if __name__ == "__main__":
demo.queue().launch() # add ?__theme=light to force light mode
