added example applications

README.md

@@ -35,6 +35,100 @@ It achieves the following results on the evaluation set:
 Here is the heatmap of the similarity score of the first 30 samples on the test split of the ROCO dataset of images vs their captions:
 ![heatmap](https://imgur.com/fPFM694.png)
 
+## Applications
+
+### Image Retrieval
+This model can be utilized for image retrieval purposes, as demonstrated below:
+
+#### Save Image Embeddings
+```
+from PIL import Image
+import pickle, os, torch
+from transformers import VisionTextDualEncoderModel, VisionTextDualEncoderProcessor
+
+# load model
+model = VisionTextDualEncoderModel.from_pretrained("kaveh/rclip")
+processor = VisionTextDualEncoderProcessor.from_pretrained("kaveh/rclip")
+
+# TO-DO
+images_path = "/path/to/images/"
+images = [os.path.join(images_path,i) for i in os.listdir(images_path) if i.endswith(".jpg")]
+
+# generate embeddings of images in your dataset
+image_embeds = []
+for img in images: 
+    with torch.no_grad():
+        inputs = processor(text=None, images=Image.open(img), return_tensors="pt", padding=True)
+        outputs = model.get_image_features(**inputs)[0].numpy()
+    image_embeds.append(outputs)
+
+# save images embeddings in a pickle file
+with open("embeddings.pkl", 'wb') as f:
+    pickle.dump(np.array(image_embeds), f)
+```
+#### Query for Images
+```
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+from PIL import Image
+import pickle
+import torch
+from transformers import VisionTextDualEncoderModel, VisionTextDualEncoderProcessor
+
+# search a query in embeddings
+query = "Chest X-Ray photos"
+
+# embed the query
+inputs = processor(text=query, images=None, return_tensors="pt", padding=True)
+with torch.no_grad():
+    query_embedding = model.get_text_features(**inputs)[0].numpy()
+
+# load image embeddings
+with open("embeddings.pkl", 'rb') as f:
+    image_embeds = pickle.load(f)
+
+# find similar images indices
+def find_k_similar_images(query_embedding, image_embeds, k=2):
+    similarities = cosine_similarity(query_embedding.reshape(1, -1), image_embeds)
+    closest_indices = np.argsort(similarities[0])[::-1][:k]
+    return closest_indices
+similar_image_indices = find_k_similar_images(query_embedding, image_embeds, k=k)
+
+# TO-DO
+images_path = "/path/to/images/"
+images = [os.path.join(images_path,i) for i in os.listdir(images_path) if i.endswith(".jpg")]
+
+# get image paths
+similar_image_names = [images[index] for index in similar_image_indices]
+Image.open(similar_image_names[0])
+```
+
+### Zero-Shot Image Classification
+
+This model can be effectively employed for zero-shot image classification, as exemplified below:
+```
+import requests
+from PIL import Image
+import matplotlib.pyplot as plt
+
+from transformers import VisionTextDualEncoderModel, VisionTextDualEncoderProcessor
+
+model = VisionTextDualEncoderModel.from_pretrained("kaveh/rclip")
+processor = VisionTextDualEncoderProcessor.from_pretrained("kaveh/rclip")
+
+url = "https://huggingface.co/spaces/kaveh/radiology-image-retrieval/resolve/main/images/ROCO_09402.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+possible_class_names = ["Chest X-Ray", "Brain MRI", "Abdominal CT Scan", "Ultrasound", "OPG"]
+
+inputs = processor(text=possible_class_names, images=image, return_tensors="pt", padding=True)
+probs = model(**inputs).logits_per_image.softmax(dim=1).squeeze()
+
+print("".join([x[0] + ": " + x[1] + "\n" for x in zip(possible_class_names, [format(prob, ".4%") for prob in probs])]))
+image
+```
+
+## Training info
+
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
@@ -98,7 +192,7 @@ The following hyperparameters were used during training:
 | 0.0974        | 4.13  | 22500 | 0.3388          |
 
 
-### Framework versions
+## Framework versions
 
 - Transformers 4.31.0.dev0
 - Pytorch 2.0.1+cu117