app.py

import os
import gradio as gr
import numpy as np
from transformers import pipeline

import torch
print(f"Is CUDA available: {torch.cuda.is_available()}")
print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")

pipe_flan = pipeline("text2text-generation", model="google/flan-t5-large", device="cuda:0")
pipe_vanilla = pipeline("text2text-generation", model="t5-large", device="cuda:0")

examples = [
  ["Please answer to the following question. Who is going to be the next Ballon d'or?"],
  ["Q: Can Geoffrey Hinton have a conversation with George Washington? Give the rationale before answering."],
  ["Please answer the following question. What is the boiling point of Nitrogen?"],
  ["Answer the following yes/no question. Can you write a whole Haiku in a single tweet?"],
  ["Answer the following yes/no question by reasoning step-by-step. Can you write a whole Haiku in a single tweet?"],
  ["Q: ( False or not False or False ) is? A: Let's think step by step"],
  ["The square root of x is the cube root of y. What is y to the power of 2, if x = 4?"],
  ["Premise:  At my age you will probably have learnt one lesson. Hypothesis:  It's not certain how many lessons you'll learn by your thirties. Does the premise entail the hypothesis?"]
]

title = "Flan T5 and Vanilla T5"
description = "This demo compares [T5-large](https://huggingface.co./t5-large) and [Flan-T5-large](https://huggingface.co./ybelkada/flan-t5-large). Note that T5 expects a very specific format of the prompts, so the examples below are not necessarily the best prompts to compare."

def inference(text):
  output_flan = pipe_flan(text, max_length=100)[0]["generated_text"]
  output_vanilla = pipe_vanilla(text, max_length=100)[0]["generated_text"]
  return [output_flan, output_vanilla]

io = gr.Interface(
  inference,
  gr.Textbox(lines=3),
  outputs=[
    gr.Textbox(lines=3, label="Flan T5"),
    gr.Textbox(lines=3, label="T5")
  ],
  title=title,
  description=description,
  examples=examples
)
io.launch()