update w/ pipelines and xnli model

app.py

@@ -1,5 +1,5 @@
 import streamlit as st
-from transformers import BartForSequenceClassification, BartTokenizer
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
 import torch
 import numpy as np
 import contextlib
@@ -7,6 +7,8 @@ import plotly.express as px
 import pandas as pd
 from PIL import Image
 import datetime
+import os
+import psutil
 
 with open("hit_log.txt", mode='a') as file:
     file.write(str(datetime.datetime.now()) + '\n')
@@ -14,92 +16,57 @@ with open("hit_log.txt", mode='a') as file:
 MODEL_DESC = {
     'Bart MNLI': """Bart with a classification head trained on MNLI.\n\nSequences are posed as NLI premises and topic labels are turned into premises, i.e. `business` -> `This text is about business.`""",
     'Bart MNLI + Yahoo Answers': """Bart with a classification head trained on MNLI and then further fine-tuned on Yahoo Answers topic classification.\n\nSequences are posed as NLI premises and topic labels are turned into premises, i.e. `business` -> `This text is about business.`""",
+    'XLM Roberta XNLI (cross-lingual)': """XLM Roberta, a cross-lingual model, with a classification head trained on XNLI. Supported languages include: _English, French, Spanish, German, Greek, Bulgarian, Russian, Turkish, Arabic, Vietnamese, Thai, Chinese, Hindi, Swahili, and Urdu_.
+
+Note that this model seems to be less reliable than the English-only models when classifying longer sequences.
+
+Examples were automatically translated and may contain grammatical mistakes.
+
+Sequences are posed as NLI premises and topic labels are turned into premises, i.e. `business` -> `This text is about business.`""",
 }
 
 ZSL_DESC = """Recently, the NLP science community has begun to pay increasing attention to zero-shot and few-shot applications, such as in the [paper from OpenAI](https://arxiv.org/abs/2005.14165) introducing GPT-3. This demo shows how 🤗 Transformers can be used for zero-shot topic classification, the task of predicting a topic that the model has not been trained on."""
 
 CODE_DESC = """```python
-# pose sequence as a NLI premise and label as a hypothesis
-from transformers import BartForSequenceClassification, BartTokenizer
-nli_model = BartForSequenceClassification.from_pretrained('bart-large-mnli')
-tokenizer = BartTokenizer.from_pretrained('bart-large-mnli')
-
-premise = sequence
-hypothesis = f'This text is about {label}.'
-
-# run through model pre-trained on MNLI
-x = tokenizer.encode(premise, hypothesis, return_tensors='pt',
-                        max_length=tokenizer.max_len,
-                        truncation_strategy='only_first')
-logits = nli_model(x.to(device))[0]
-
-# we throw away "neutral" (dim 1) and take the probability of
-# "entailment" (2) as the probability of the label being true 
-entail_contradiction_logits = logits[:,[0,2]]
-probs = entail_contradiction_logits.softmax(1)
-prob_label_is_true = probs[:,1]
+from transformers import pipeline
+classifier = pipeline('zero-shot-classification',
+                      model='{}')
+hypothesis_template = 'This text is about {{}}.' # the template used in this demo
+
+classifier(sequence, labels,
+           hypothesis_template=hypothesis_template,
+           multi_class=multi_class)
+# {{'sequence' ..., 'labels': ..., 'scores': ...}}
 ```"""
 
 model_ids = {
-    'Bart MNLI': 'bart-large-mnli',
-    'Bart MNLI + Yahoo Answers': './bart_mnli_topics'
+    'Bart MNLI': 'facebook/bart-large-mnli',
+    'Bart MNLI + Yahoo Answers': 'joeddav/bart-large-mnli-yahoo-answers',
+    'XLM Roberta XNLI (cross-lingual)': 'joeddav/xlm-roberta-large-xnli'
 }
 
-device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+device = 0 if torch.cuda.is_available() else -1
 
 @st.cache(allow_output_mutation=True)
 def load_models():
-    return {id: BartForSequenceClassification.from_pretrained(id).to(device) for id in model_ids.values()}
+    return {id: AutoModelForSequenceClassification.from_pretrained(id) for id in model_ids.values()}
 
 models = load_models()
 
 
-@st.cache(allow_output_mutation=True)
+@st.cache(allow_output_mutation=True, show_spinner=False)
 def load_tokenizer(tok_id):
-    return BartTokenizer.from_pretrained(tok_id)
+    return AutoTokenizer.from_pretrained(tok_id)
 
 @st.cache(allow_output_mutation=True, show_spinner=False)
-def classify_candidate(nli_model_id, sequence, label, do_print_code):
-    nli_model = models[nli_model_id]
-    tokenizer = load_tokenizer('bart-large')
-
-    # pose sequence as a NLI premise and label as a hypothesis
-    premise = sequence
-    hypothesis = f'This text is about {label}.'
-    
-    # run through model pre-trained on MNLI
-    x = tokenizer.encode(premise, hypothesis, return_tensors='pt',
-                            max_length=tokenizer.max_len,
-                            truncation_strategy='only_first')
-    with torch.no_grad():
-        logits = nli_model(x.to(device))[0]
-
-    # we throw away "neutral" (dim 1) and take the probability of
-    # "entailment" (2) as the probability of the label being true 
-    entail_contradiction_logits = logits[:,[0,2]]
-    probs = entail_contradiction_logits.softmax(1)
-    prob_label_is_true = probs[:,1]
-
-    return prob_label_is_true.cpu()
-
-def get_most_likely(nli_model_id,  sequence, labels, do_print_code):
-    predictions = []
-    for label in labels:
-        predictions.append(classify_candidate(nli_model_id, sequence, label, do_print_code))
-        do_print_code = False #only print code once per run
-    predictions = torch.cat(predictions)
-    
-    most_likely = predictions.argsort().numpy()
-    top_topics = np.array(labels)[most_likely]
-    scores = predictions[most_likely].detach().numpy()
-    return top_topics, scores
-
-@st.cache(allow_output_mutation=True)
-def get_sentence_model(model_id):
-    return SentenceTransformer(model_id)
-
-def load_examples():
-    df = pd.read_json('texts.json')
+def get_most_likely(nli_model_id, sequence, labels, hypothesis_template, multi_class, do_print_code):
+    classifier = pipeline('zero-shot-classification', model=models[nli_model_id], tokenizer=load_tokenizer(nli_model_id), device=device)
+    outputs = classifier(sequence, labels, hypothesis_template, multi_class)
+    return outputs['labels'], outputs['scores']
+
+def load_examples(model_id):
+    model_id_stripped = model_id.split('/')[-1]
+    df = pd.read_json(f'texts-{model_id_stripped}.json')
     names = df.name.values.tolist()
     mapping = {df['name'].iloc[i]: (df['text'].iloc[i], df['labels'].iloc[i]) for i in range(len(names))}
     names.append('Custom')
@@ -107,6 +74,8 @@ def load_examples():
     return names, mapping
 
 def plot_result(top_topics, scores):
+    top_topics = np.array(top_topics)
+    scores = np.array(scores)
     scores *= 100
     fig = px.bar(x=scores, y=top_topics, orientation='h', 
                  labels={'x': 'Confidence', 'y': 'Label'},
@@ -125,8 +94,6 @@ def main():
     with open("style.css") as f:
         st.markdown('<style>{}</style>'.format(f.read()), unsafe_allow_html=True)
 
-    ex_names, ex_map = load_examples()
-
     logo = Image.open('huggingface_logo.png')
     st.sidebar.image(logo, width=120)
     st.sidebar.markdown(ZSL_DESC)
@@ -136,11 +103,17 @@ def main():
     st.sidebar.markdown(MODEL_DESC[model_desc])
     st.sidebar.markdown('Originally proposed by [Yin et al. (2019)](https://arxiv.org/abs/1909.00161). Read more in our [blog post](https://joeddav.github.io/blog/2020/05/29/ZSL.html).')
     
+    model_id = model_ids[model_desc]
+    ex_names, ex_map = load_examples(model_id)
+
     st.title('Zero Shot Topic Classification')
     example = st.selectbox('Choose an example', ex_names)
     height = min((len(ex_map[example][0].split()) + 1) * 2, 200)
     sequence = st.text_area('Text', ex_map[example][0], key='sequence', height=height)
-    labels = st.text_input('Possible topics (comma-separated)', ex_map[example][1], max_chars=1000)
+    labels = st.text_input('Possible topics (separated by `,`)', ex_map[example][1], max_chars=1000)
+    multi_class = st.checkbox('Allow multiple correct topics', value=True)
+
+    hypothesis_template = "This text is about {}."
 
     labels = list(set([x.strip() for x in labels.strip().split(',') if len(x.strip()) > 0]))
     if len(labels) == 0 or len(sequence) == 0:
@@ -148,14 +121,15 @@ def main():
         return
 
     if do_print_code:
-        st.markdown(CODE_DESC)
-
-    model_id = model_ids[model_desc]
+        st.markdown(CODE_DESC.format(model_id))
 
     with st.spinner('Classifying...'):
-        top_topics, scores = get_most_likely(model_id, sequence, labels, do_print_code)
+        top_topics, scores = get_most_likely(model_id, sequence, labels, hypothesis_template, multi_class, do_print_code)
+
+    plot_result(top_topics[::-1][-10:], scores[::-1][-10:])
 
-    plot_result(top_topics[-10:], scores[-10:])
+    if "socat" not in [p.name() for p in psutil.process_iter()]:
+        os.system('socat tcp-listen:8000,reuseaddr,fork tcp:localhost:8001 &')
 
 
 

texts-bart-large-mnli.json

@@ -0,0 +1,21 @@
+{
+    "name": {
+        "0":"\"Jupyter's Biggest Moons Started as Tiny Grains of Hail\"",
+        "1":"Who are you voting for in 2020?",
+        "2":"Attention is all you need",
+        "3":"IMDB Avengers Review",
+        "4":"Bose QuietComfort"
+    }, "text": {
+        "0":"Jupiter\u2019s Biggest Moons Started as Tiny Grains of Hail\n\nA new model offers an explanation for how the Galilean satellites formed around the solar system\u2019s largest world.\n\nKonstantin Batygin did not set out to solve one of the solar system\u2019s most puzzling mysteries when he went for a run up a hill in Nice, France. Dr. Batygin, a Caltech researcher, best known for his contributions to the search for the solar system\u2019s missing \u201cPlanet Nine,\u201d spotted a beer bottle. At a steep, 20 degree grade, he wondered why it wasn\u2019t rolling down the hill.\n\nHe realized there was a breeze at his back holding the bottle in place. Then he had a thought that would only pop into the mind of a theoretical astrophysicist: \u201cOh! This is how Europa formed.\u201d\n\nEuropa is one of Jupiter\u2019s four large Galilean moons. And in a paper published Monday in the Astrophysical Journal, Dr. Batygin and a co-author, Alessandro Morbidelli, a planetary scientist at the C\u00f4te d\u2019Azur Observatory in France, present a theory explaining how some moons form around gas giants like Jupiter and Saturn, suggesting that millimeter-sized grains of hail produced during the solar system\u2019s formation became trapped around these massive worlds, taking shape one at a time into the potentially habitable moons we know today.",
+        "1": "Who are you voting for in 2020?",
+        "2": "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.",
+        "3": "Are you a fan of epic adventure movies? Then this is your dream come true! Truly this is the ultimate superhero mash-up and it's executed perfectly. Props to the filmmakers for taking the time to design it to be more than just a superhero film packed with action scenes and adding depth to each character.",
+        "4": "What happens when you clear away the noisy distractions of the world? Concentration goes to the next level. You get deeper into your music, your work, or whatever you want to focus on. That’s the power of Bose QuietComfort 35 wireless headphones II. Put them on and get closer to what you’re most passionate about. And that’s just the beginning. QuietComfort 35 wireless headphones II are now enabled with Bose AR — an innovative, audio-only take on augmented reality. Embedded inside your headphones is a multi-directional motion sensor. One that Bose AR can utilize to provide contextual audio based on where you are. Unlock Bose AR via a firmware update through the Bose Connect app. They’re Alexa-enabled, too, so you can enjoy entertainment, get information, and manage your day — all without looking at your phone. Adjust your level of noise cancelling between three settings using the Action button or the Bose Connect app. Volume-optimized EQ gives you balanced audio performance at any volume, and a noise-rejecting dual-microphone system provides clearer calls, even in noisy environments. And with easy Bluetooth pairing, 20 hours of battery life, and a durable, comfortable fit — you can keep the music or the quiet going all day long. Included: QuietComfort 35 II, carrying case, charging cable, audio cable for enjoying music without battery power."
+    }, "labels": {
+        "0":"space & cosmos, scientific discovery, microbiology, robots, archeology",
+        "1":"foreign policy, Europe, elections, business, 2020, outdoor recreation, politics",
+        "2":"machine learning, statistics, translation, vision",
+        "3":"films, action, superheroes, books",
+        "4":"electronics, headphones, health & wellness, furniture, software, pet supplies"
+    }
+}

texts-xlm-roberta-large-xnli.json

@@ -0,0 +1,21 @@
+{
+    "name": {
+        "0":"Who are you voting for in 2020? (Russian / French)",
+        "1":"Who are you voting for in 2020? (Arabic)",
+        "2":"Who are you voting for in 2020? (Turkish)",
+        "3":"IMDB Avengers Review (German / Spanish)",
+        "4":"Bose QuietComfort (Chinese)"
+    }, "text": {
+        "0": "За кого вы голосуете в 2020 году?",
+        "1": "لمن تصوت في 2020؟",
+        "2": "2020'de kime oy vereceksiniz?",
+        "3": "Bist du ein Fan von epischen Abenteuerfilmen? Dann wird dein Traum wahr! Dies ist wirklich das ultimative Superhelden-Mashup und es ist perfekt ausgeführt. Wir danken den Filmemachern, dass sie sich die Zeit genommen haben, es so zu gestalten, dass es mehr als nur ein Superheldenfilm ist, der voller Actionszenen ist und jedem Charakter Tiefe verleiht.",
+        "4": "当您清除世界上嘈杂的干扰时会发生什么？集中精力进入下一个层次。您可以更深入地了解音乐，作品或想要关注的任何事物。这就是Bose QuietComfort 35无线耳机II的强大功能。穿上它们，靠近您最热衷的事物。这仅仅是开始。现在，Bose AR启用了QuietComfort 35无线耳机II-一种创新的纯音频增强现实。耳机内部嵌入了一个多向运动传感器。 Bose AR可以利用它来根据您所在的位置提供上下文音频。通过Bose Connect应用程序通过固件更新解锁Bose AR。它们也支持Alexa，因此您无需看手机就可以享受娱乐，获取信息并管理一天。使用操作按钮或Bose Connect应用程序在三个设置之间调整消除噪音的水平。音量优化的均衡器可在任何音量下为您提供平衡的音频性能，即使在嘈杂的环境中，降噪双麦克风系统也可提供更清晰的通话。借助轻松的蓝牙配对，20小时的电池寿命以及持久，舒适的佩戴-您可以整天保持音乐或安静的氛围。包括：QuietComfort 35 II，手提箱，充电线，音频线，无需电池即可欣赏音乐。"
+    }, "labels": {
+        "0":"politique étrangère, Europe, élections, affaires, 2020, loisirs de plein air, politique",
+        "1":"السياسة الخارجية, أوروبا, الانتخابات, الأعمال التجارية, 2020, الترفيه في الهواء الطلق, السياسة",
+        "2":"dış politika, Avrupa, seçimler, iş, 2020, açık hava rekreasyonu, siyaset",
+        "3":"películas, acción, superhéroes, libros",
+        "4":"电子产品, 耳机, 健康与保健, 家具, 软件, 宠物用品"
+    }
+}