index.sh

#! /usr/bin/env bash

#pull images url from the web
python imgs.py
local_data_folder=$(python -c "from app._config import settings; print(settings.LOCAL_DATA_FOLDER)")

cd "$local_data_folder"
#remove if folders exists
rm -rf embeddings_folder
rm -rf image_folder
rm -rf index_folder
rm -rf retrieved_folder

echo "download image urls into image folder"
img2dataset --url_list=myimglist.txt --output_folder=image_folder --thread_count=64 --image_size=256

echo "create embedding folder"
#change --num_prepro_workers > 0 to enable multiprocessing
clip-retrieval inference --input_dataset image_folder --output_folder embeddings_folder --enable_text False --num_prepro_workers 0

echo "create indices from embedding folder"
clip-retrieval index --embeddings_folder embeddings_folder --index_folder index_folder