Duplicate from YoannLemesle/CLIPictionary

Co-authored-by: Yoann Lemesle <[email protected]>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: CLIPictionary!
+emoji: ✏️
+colorFrom: yellow
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.0.12
+app_file: app.py
+pinned: true
+duplicated_from: YoannLemesle/CLIPictionary
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,47 @@
+import matplotlib.pyplot as plt
+import torch, torchvision, clip, random
+import random, math, time, os
+from PIL import Image
+import gradio as gr
+import numpy as np
+
+from sentence import *
+from game import *
+
+##### Initialize new game
+title, _,_,var_dict    = new_game(None, None, first_game=True)
+var_dict["start_time"] = -1
+
+##### Display & Events
+demo = gr.Blocks()
+with demo:
+    ### All game variables are stored here
+    variables = gr.Variable(var_dict)
+    ### Target Sentence
+    title = gr.HTML(title)
+    ### Canvas & Prediction
+    with gr.Column():
+        with gr.Row():
+            image_input = gr.Image(image_mode='L', label="", show_label=False, source='canvas', shape=None, streaming=False, invert_colors=False, tool="editor")
+            with gr.Column():
+                html_pred    = gr.HTML(value=getHTML(var_dict,""))
+                html_loading = gr.HTML("")
+        ### 'New Sentence' Button
+        with gr.Row():
+            button_new   = gr.Button("New Sentence",variant="primary")
+            button_mode  = gr.Button("Switch Difficulty")
+            block        = gr.HTML("<div style=\"width:50vw;\"> </div>")
+        ### Informations
+        gr.HTML("<div style=\"display:block; height:30px;\"> </div>")
+        with gr.Row():
+            gr.HTML("<div style=\"display:block; position:relative; bottom:10%; border-top: 1px solid grey;  padding:10px; \"><span style=\"font-size:30px;\">✏️</span><span style=\"font-size:40px; font-weight:bold;\">CLIPictionary!</span><br>Draw to make the model guess the target sentence displayed at the top!<br>Made by <a href=\"https://yoann-lemesle.notion.site/Yoann-Lemesle-63b8120764284794b275d2967be710da\" style=\"text-decoration: underline;\">Yoann Lemesle</a> using OpenAI's <a href=\"https://github.com/openai/CLIP\" style=\"text-decoration: underline;\">CLIP model</a>.</div>")
+
+
+    ### Events
+    button_mode.click(switch_difficulty,inputs=[variables,html_loading],outputs=[variables,title,html_pred,html_loading])
+    button_new.click(loading,inputs=html_loading,outputs=[title,html_pred,html_loading])  # Button -> triggers Loading
+    html_loading.change(new_game,inputs=[variables,html_loading],outputs=[title,html_pred,image_input,variables])     # Loading -> triggers New game
+    image_input.change(process_img, inputs=[variables,image_input,title], outputs=[html_pred,title,variables])
+
+
+demo.launch(share=False)

game.py

@@ -0,0 +1,140 @@
+import torch, torchvision, clip, time, math
+import matplotlib.pyplot as plt
+from model import encoder_image
+from sentence import *
+
+##### Get infos & cool facts to display during loadings
+with open("infos.txt") as file:
+    infos = file.readlines()
+
+##### Get css
+with open("style.css") as style:
+    css = "<style>"+ ''.join(style.readlines())+"</style>"
+
+##### 'DIFFICULTY SWITCH' EVENT
+def switch_difficulty(var_dict, html_loading):
+
+    var_dict["difficulty"] = 1 - var_dict["difficulty"]
+
+    title, infos, new_value = loading(html_loading)
+    return var_dict, title, infos, new_value
+
+##### 'LOADING' EVENT
+def loading(html_loading=None):
+    ### This is just to make sure the content changes, which triggers the .change event which, itself, will launch a new game
+    if html_loading == "<div style=\"display:none;\">0</div>": new_value = "<div style=\"display:none;\">1</div>"
+    else:                                                      new_value = "<div style=\"display:none;\">0</div>"
+    ### Get a random tip
+    info = np.random.choice(infos)
+    ### Return TITLE, TIP TEXT, NEW HTML CONTENT, CANVAS IMG
+    return "<h1 id=\"loading\">⌛Loading...</h1>",css+"<div id=\"prediction\"><p id=\"infos\">"+info+"</p></div>",new_value
+
+##### 'NEW GAME' EVENT
+def new_game(var_dict,img=None,first_game=False):
+    print("\n----------Launching new game!")
+
+    if None is not var_dict:    difficulty = var_dict["difficulty"]
+    else:                       difficulty = 1
+
+    var_dict = {
+        "start_time":           time.time(),
+        "total_time":           0,
+        "found_words":          [],
+        "target_sentence":      "",
+        "guessed_sentence":     "",
+        "parts":                [],
+        "win":                  0,
+        "step":                 0,
+        "prev_steps":           [],
+        "prev_norm":            float("inf"),
+        "tip":                  "",
+        "loading":              False,
+        "revertedState":        False,
+        "difficulty":           difficulty
+    }
+    target = iniSentence(var_dict,first_game=first_game)
+    ### Return TITLE, PREDICTION TEXT, CANVAS IMG, VAR DICT
+    return "<h1>"+target+"</h1>", getHTML(var_dict,""), None, var_dict
+
+##### PREDICTION TEXT HTML
+def getHTML(var_dict,text,win=0):
+    ### Which parts of the sentence have been guessed?
+    guessed, not_guessed = "", ""
+    text_words           = text.split(" ")
+    target_words         = var_dict["target_sentence"].split(" ")
+    for i,word in enumerate(text_words):
+        if i < len(target_words) and word == target_words[i]: guessed += word + " "
+        else:                                                 not_guessed += word + " "
+    ### Display prediction
+    if win!=1:
+        html = "<p><span>"+guessed+"</span>"+not_guessed+"</p>"
+    else:
+        minutes, seconds  = math.floor(var_dict["total_time"]/60), var_dict["total_time"]%60
+        if minutes < 1 and seconds <= 30:   emoji = "🏆😍"
+        elif minutes < 1:                   emoji = "😄"
+        elif minutes < 2:                   emoji = "😐"
+        elif minutes < 3:                   emoji = "😓"
+        else:                               emoji = "😱"
+        time_str = "Total time: "+ ((str(minutes)+"m") if minutes>0 else "") + str(seconds)+"s "+emoji
+        html     = "<p id=\"win\"><span>"+guessed+"</span><br>"+time_str+"</p>"
+    return css+"<div id=\"prediction\">"+html+"</div>"
+
+##### DRAWING PROCESSING & GAME STATE UPDATE
+def process_img(var_dict,img,title):
+    # Makes sure that start_time is updates for the first game
+    if var_dict["start_time"] == -1:
+        var_dict["start_time"] = time.time()
+    if (None is img):
+        return getHTML(var_dict,"",win=0),"<h1>"+var_dict["target_sentence"]+"</h1>",var_dict
+    elif (None is not img) and (var_dict["win"] != 1):
+        print("-----Processing...")
+        part   = var_dict["parts"][var_dict["step"]]
+        image = torch.tensor(img).float() / 255
+
+        ### Detect Cancel event
+        norm  = torch.norm(image)
+        if norm > var_dict["prev_norm"]:
+            print("---Cancel Event")
+            prevState(var_dict)
+        var_dict["prev_norm"] = norm
+
+        ### Image preprocessing --> shape (224,224)
+        max_edge = max(image.shape[0],image.shape[1])
+        min_edge = min(image.shape[0],image.shape[1])
+        square_image  = torch.ones(max_edge,max_edge)
+        pad           = math.floor((max_edge - min_edge)/2)
+        if max_edge == image.shape[1]: square_image[pad:pad+min_edge,:] = image
+        else:                          square_image[:,pad:pad+min_edge] = image
+        image = torchvision.transforms.Resize((224,224))(square_image.unsqueeze(0)).repeat(1,3,1,1)
+
+        ### Computing cosine similarities (drawing<->text embeddings)
+        with torch.no_grad():
+            image_features = encoder_image(image)[0]
+            text_features  = torch.tensor(part["embeddings"])
+            image_features /= image_features.norm()
+            similarities   = torch.matmul(text_features,image_features)
+            probs          = torch.nn.Softmax(dim=-1)(similarities)
+
+        ### Sort indexes by similarity
+        idxs   = np.argsort(similarities)
+
+        ### Use top-3 preditions
+        top3_idxs = idxs[-3:]
+        classes   = part["classes"]
+        preds     = [classes[idx] for idx in top3_idxs]
+        print(f"Top-3 Predictions: {preds}")
+        print(f"Top-3 Probabilities: {probs[top3_idxs]}")
+
+        ### Check if win (-1: bad guess, 0:progress=guessed sentence part, 1:win=guessed whole sentence)
+        win = updateState(var_dict, preds)
+        if win == -1:
+            text = preds[-1]
+        elif win == 0:
+            part = var_dict["parts"][var_dict["step"]]
+            text = var_dict["guessed_sentence"] + link_text(part,"something") + " something"
+        elif win == 1:
+            text = var_dict["guessed_sentence"]
+            if var_dict["total_time"] == 0: var_dict["total_time"] = round(time.time() - var_dict["start_time"])
+        return getHTML(var_dict,text,var_dict["win"]),"<h1>"+var_dict["target_sentence"]+"</h1>",var_dict
+    else:
+        return getHTML(var_dict,var_dict["target_sentence"],win=1),"<h1>"+var_dict["target_sentence"]+"</h1>",var_dict

infos.txt

@@ -0,0 +1,12 @@
+Tip: Filling shapes can help!
+Tip: Drawing context elements can help!
+Tip: Big drawings are more likely to work!
+Tip: Adding a face to objects doing actions helps!
+Tip: Add a musical note to help CLIP guess instruments!
+Tip: Add a fork to help CLIP guess food!
+Did you know? CLIP has multimodal neurons!
+Did you know? The paper introducing CLIP has been published in 2021.
+Did you know? CLIP stands for 'Contrastive Language-Image Pretraining'.
+Did you know? DALL·E 2 can generate images from text using CLIP's embeddings!
+Did you know? Typographic attacks (adding text to images) can change CLIP's predictions!
+Did you know? CLIP has been trained on 400 millions (image,text) pairs!

model.py

@@ -0,0 +1,8 @@
+##### Initializes CLIP (from https://github.com/openai/CLIP)
+import torch, torchvision, clip
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model, preprocess = clip.load("ViT-B/32", device=device)
+
+encoder_image = model.encode_image
+encoder_text  = model.encode_text

requirements.txt

@@ -0,0 +1,7 @@
+scipy
+numpy
+torch
+ftfy
+regex
+tqdm
+git+https://github.com/openai/CLIP.git

sentence.py

@@ -0,0 +1,115 @@
+from model import encoder_text
+import torch, clip, random
+import numpy as np
+device = torch.device("cpu")
+
+from words import words
+
+########## SENTENCE PART #######################################################
+voyelles = ["a","e","i","o","u"]
+links    = list(words.keys())[1:]
+
+def link_text(part,nextWord):
+    ### Check if we need to write "... a", "... an", "..."
+    if (len(part["link"]) > 0) and (part["link"][-1] == "a"):
+        voyelleStart = (nextWord[0] in voyelles)
+        plural       = (nextWord[-1] == "s" and nextWord[-2] != "s") or (nextWord in ["nothing","hair","vampire teeth","something"])
+    else:
+        voyelleStart, plural = False, False
+    return (part["link"][:-2] if plural else part["link"] + ("n" if voyelleStart else ""))
+
+def part_text(part):
+    l = link_text(part,part["word"])
+    return l + (" " if len(l)>0 else "") + part["word"]
+
+def compute_embeddings(part,var_dict,prefix,batch_size=64):
+    target        = part["word"]
+    possibleWords = list(set(words[part["link"]]) - set([target]+var_dict["found_words"]))
+    if len(possibleWords) > (batch_size-1): possibleWords = np.random.choice(list(possibleWords),batch_size-1,replace=False).tolist()
+    possibleWords.append(target)
+    ### Compute all classes & embeddings for current sentence part
+    part["classes"] = [prefix + link_text(part,w) + (" " if len(link_text(part,w))>0 else "") + w for w in possibleWords]
+    with torch.no_grad():
+        embeddings         = encoder_text(clip.tokenize(part["classes"]).to(device))
+        embeddings        /= embeddings.norm(dim=-1, keepdim=True)
+        part["embeddings"] = embeddings.tolist()
+
+########## SENTENCE ############################################################
+def iniSentence(var_dict,input="",first_game=False):
+    var_dict["found_words"] = []
+    var_dict["parts"]       = []
+    var_dict["step"]        = 0
+    prefix                  = ""
+    N                       = (2 if var_dict["difficulty"] == 1 else 1)
+
+    if first_game:
+        link = "a drawing of a"
+        part = {"link":link,"word":"cat","classes":[],"embeddings":[]}
+        var_dict["parts"].append(part)
+        compute_embeddings(part, var_dict, prefix)
+        prefix += part_text(part) + " "
+
+        link = "with a"
+        part = {"link":link,"word":"face","classes":[],"embeddings":[]}
+        var_dict["parts"].append(part)
+        compute_embeddings(part, var_dict, prefix)
+        prefix += part_text(part) + " "
+    else:
+        ##### Generating Random Sentence
+        link = "a drawing of a"
+        part = {"link":link,"word":np.random.choice(words[link]),"classes":[],"embeddings":[]}
+        var_dict["parts"].append(part)
+        compute_embeddings(part, var_dict, prefix)
+        prefix += part_text(part) + " "
+
+        for i in range(N-1):
+            link  = np.random.choice(links)
+            part  = {"link":link,"word":np.random.choice(words[link][1:]),"classes":[],"embeddings":[]}
+            var_dict["parts"].append(part)
+            compute_embeddings(part, var_dict, prefix)
+            prefix += part_text(part) + " "
+
+    var_dict["target_sentence"] = prefix[:-1] # Target sentence is prefix without the last space
+    setState(var_dict)
+    return var_dict["target_sentence"]
+
+def prevState(var_dict):
+    if len(var_dict["prev_steps"]) > 0: var_dict["step"] = var_dict["prev_steps"].pop(-1)
+    else:                               var_dict["step"] = 0
+    var_dict["revertedState"] = True
+    setState(var_dict)
+
+def setState(var_dict):
+    var_dict["found_words"] = var_dict["found_words"][:var_dict["step"]]
+    var_dict["guessed_sentence"] = ""
+    for i in range(var_dict["step"]):
+        var_dict["guessed_sentence"] += part_text(var_dict["parts"][i]) + " "
+
+def updateState(var_dict, preds):
+    if not var_dict["revertedState"]:  var_dict["prev_steps"].append(var_dict["step"])
+    else:                              var_dict["revertedState"] = False
+
+    ### Check if the current part has been guessed
+    part = var_dict["parts"][var_dict["step"]]
+
+    
+    idx_of_nothing = -1
+    if ("nothing" in preds[0]):   idx_of_nothing = 0
+    elif ("nothing" in preds[1]): idx_of_nothing = 1
+    elif ("nothing" in preds[2]): idx_of_nothing = 2
+
+    idx_of_guess = -1
+    if (part["classes"][-1] == preds[0]):   idx_of_guess = 0
+    elif (part["classes"][-1] == preds[1]): idx_of_guess = 1
+    elif (part["classes"][-1] == preds[2]): idx_of_guess = 2
+
+
+    if not var_dict["win"] and (idx_of_guess > idx_of_nothing):
+        var_dict["step"] += 1
+        var_dict["found_words"].append(part["word"])
+        var_dict["win"] = var_dict["step"] == len(var_dict["parts"])
+        setState(var_dict)
+        if var_dict["win"]: return 1
+        else:               return 0
+    elif not var_dict["win"]: return -1
+    else:                     return 1

style.css

@@ -0,0 +1,128 @@
+
+@-webkit-keyframes winAnim {
+  from { text-shadow:0px 0px 15px rgb(250,150,0,0); font-size:25px;}
+  to { text-shadow:0px 0px 15px rgb(250,150,0,0.75); font-size:30px;}
+}
+
+@keyframes winAnim {
+  from { text-shadow:0px 0px 15px rgb(250,150,0,0); font-size:25px;}
+  to { text-shadow:0px 0px 15px rgb(250,150,0,0.75); font-size:30px;}
+}
+
+
+#test{
+  font-weight:bold;
+}
+
+@-webkit-keyframes prediction {
+  from { opacity:0;}
+  to { opacity:1;}
+}
+
+@keyframes prediction {
+  from { opacity:0;}
+  to { opacity:1;}
+}
+
+
+div#prediction{
+  display:block;
+  width:45vw;
+  height:100%;
+  text-align:center;
+  margin-top:100px;
+  margin-left:auto;
+  margin-right: auto;
+}
+
+div#prediction p:not(#win){
+  font-size:25px;
+  display:inline-block;
+  margin:auto;
+  position:relative;
+  top:48%;
+  position-anchor: 50% 50%;
+  text-align:center;
+  transform: translate(0, -50%);
+  -webkit-animation: prediction 0.25s;
+  animation: prediction 0.25s;
+}
+
+div#prediction p#win{
+  font-weight:bold;
+  -webkit-animation: winAnim 0.5s;
+  animation: winAnim 0.5s;
+  font-size:30px;
+  text-shadow:0px 0px 15px rgb(250,150,0,0.75);
+}
+
+div#prediction p#infos{
+  font-weight:bold;
+  -webkit-animation: winAnim 0.5s;
+  animation: winAnim 0.5s;
+  font-size:30px;
+  text-shadow:0px 0px 15px rgb(250,150,0,0.75);
+}
+
+
+span{
+  font-weight:bold;
+  color:rgb(250,150,0);
+}
+
+h1{
+  display:block; font-size:30px; font-weight:bold; width:100%; text-align:center; margin-bottom:15px;
+}
+
+
+@-webkit-keyframes loading {
+  from { color:rgb(250,150,0,0.25);}
+  to { color:rgb(250,150,0,1);}
+}
+
+@keyframes loading {
+  from { color:rgb(250,150,0,0.25);}
+  to { color:rgb(250,150,0,1);}
+}
+
+
+h1#loading{
+  margin-bottom:15px;
+  display:block;
+  width:100%;
+  vertical-align: center;
+  text-align: center;
+
+  font-size:30px;
+  font-weight:bold;
+
+  -webkit-animation: loading 10s;
+  animation: loading 10s;
+  color:rgb(250,150,0,1);
+}
+
+@-webkit-keyframes processing {
+  from { opacity:1;}
+  to { opacity:0;}
+}
+
+@keyframes processing {
+  from { opacity:1;}
+  to { opacity:0;}
+}
+
+
+p#processing{
+
+  -webkit-animation: loading 3s;
+  animation: loading 3s;
+  color:rgb(250,150,0);
+  font-weight:bold;
+
+  font-size:50px;
+  display:block;
+  margin:auto;
+  text-align:center;
+  width:100%;
+
+}

words.py

@@ -0,0 +1,32 @@
+################################################################################
+# A DRAWING OF A ...
+shapes      = ["triangle","square","circle","heart","star","diamond"]
+animals     = ["cat","dog","duck","bee","butterfly","bird","pig","cow","fish","frog","shark","snake","mouse","monkey","snail"]
+objects     = ["bottle","wine glass","plane","spoon","basket ball","chair","pen","computer","hat","soccer ball","phone","sword","axe","umbrella","bell","dumbbell","scissors","fork","bag","clock","key","shopping cart","car","boat","house","mug","sun","moon","atom","hand"]
+plants      = ["tree","flower","leaf","palm tree","mushroom"] # I know that mushrooms are not plants stop coming to my house
+food        = ["donut","coconut","banana","apple","sausage","icecream","burger","egg","pizza"]
+instruments = ["drum","guitar","piano","flute","trumpet","accordion"]
+
+# FEELING ...
+feelings = ["neutral","happy","sad","angry","surprised","thirsty","sleepy","hungry","love","curious","evil"]
+
+# WITH A...
+attributes = ["face","mustache","muscles","vampire teeth","hair","eye","mouth","scar"]
+
+# WEARING A...
+clothings  = ["glasses","sunglasses","hat","socks","eye patch","pants","tee-shirt","scarf"]
+
+# ...
+others = ["that is talking","that is dancing","that is singing","and its clone","at the beach","at a forest"]
+
+words = {
+    "a drawing of a": shapes + animals + objects + instruments + plants + food,
+    "with a":         ["nothing"]+attributes + clothings,
+    "wearing a":      ["nothing"]+clothings,
+    "eating a":       ["nothing"]+food,
+    "playing the":    ["nothing"]+instruments,
+    "holding a":      ["nothing"]+shapes + animals + objects + instruments + plants + food,
+    "feeling":        ["nothing"]+feelings,
+    "":               ["and nothing else"]+others
+    }
+################################################################################