Upload folder using huggingface_hub

.gitignore

@@ -178,3 +178,4 @@ coverage_report/
 local_data/
 models/
 .DS_Store
+/app/ui/multimodalchatbot

README.md

@@ -1,8 +1,8 @@
 ---
 title: discord-bot
-app_file: app/main.py
+app_file: app/__main__.py
 sdk: gradio
-sdk_version: 4.33.0
+sdk_version: 4.26.0
 ---
 # Capstone Project
 
@@ -40,6 +40,10 @@ Download embedding and(or) LLM models
 ```shell
 bash prestart.sh
 ```
+Download web images and creating indices
+```shell
+bash index.sh
+```
 
 ### Install `pre-commit` hooks
 

app/_config.py

@@ -15,8 +15,9 @@ class Settings(BaseSettings):
 
     WEAVIATE_CLIENT_URL: str = "http://localhost:8080"
 
-    LLM_MODE: Literal["openai", "mock", "local"] = "mock"
-    EMBEDDING_MODE: Literal["openai", "mock", "local"] = "mock"
+    LLM_MODE: Literal["openai", "mock", "local"] = "local"
+    EMBEDDING_MODE: Literal["openai", "mock", "local"] = "local"
+    IMG_DATASET: Literal["growstuff"] = "growstuff"
 
     LOCAL_DATA_FOLDER: str = "local_data/test"
 
@@ -44,11 +45,6 @@ class Settings(BaseSettings):
     IS_UI_ENABLED: bool = True
     UI_PATH: str = "/"
 
-    # Rerank
-    IS_RERANK_ENABLED: bool = True
-    RERANK_TOP_N: int = 3
-    RERANK_MODEL_NAME: str = "cross-encoder/ms-marco-MiniLM-L-2-v2"
-
     class Config:
         case_sensitive = True
         env_file_encoding = "utf-8"

app/components/llm/component.py

@@ -42,7 +42,7 @@ class LLMComponent:
                     # set to at least 1 to use GPU
                     # set to -1 for all gpu
                     # set to 0 for cpu
-                    model_kwargs={"n_gpu_layers": 0},
+                    model_kwargs={"n_gpu_layers": -1},
                     # transform inputs into Llama2 format
                     messages_to_prompt=messages_to_prompt,
                     completion_to_prompt=completion_to_prompt,

app/main.py

@@ -1,7 +1,6 @@
 import logging
 
 from fastapi import FastAPI
-import gradio as gr
 
 from app._config import settings
 from app.components.embedding.component import EmbeddingComponent
@@ -37,6 +36,3 @@ if settings.IS_UI_ENABLED:
 
     ui = PrivateGptUi(ingest_service, chat_service)
     ui.mount_in_app(app, settings.UI_PATH)
-    
-    io = gr.Interface(lambda x: "Hello, " + x + "!", "textbox", "textbox")
-    app = gr.mount_gradio_app(app, io, settings.UI_PATH)        

app/server/chat/service.py

@@ -3,11 +3,9 @@ from dataclasses import dataclass
 from llama_index import ServiceContext, StorageContext, VectorStoreIndex
 from llama_index.chat_engine import ContextChatEngine
 from llama_index.chat_engine.types import BaseChatEngine
-from llama_index.core.postprocessor import SentenceTransformerRerank
 from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
 from llama_index.llms import ChatMessage, MessageRole
 
-from app._config import settings
 from app.components.embedding.component import EmbeddingComponent
 from app.components.llm.component import LLMComponent
 from app.components.node_store.component import NodeStoreComponent
@@ -79,20 +77,13 @@ class ChatService:
             index=self.index
         )
 
-        node_postprocessors = [
-            MetadataReplacementPostProcessor(target_metadata_key="window")
-        ]
-        if settings.IS_RERANK_ENABLED:
-            rerank = SentenceTransformerRerank(
-                top_n=settings.RERANK_TOP_N, model=settings.RERANK_MODEL_NAME
-            )
-            node_postprocessors.append(rerank)
-
         return ContextChatEngine.from_defaults(
             system_prompt=system_prompt,
             retriever=vector_index_retriever,
             service_context=self.service_context,
-            node_postprocessors=node_postprocessors,
+            node_postprocessors=[
+                MetadataReplacementPostProcessor(target_metadata_key="window"),
+            ],
         )
 
     def chat(self, messages: list[ChatMessage]):

app/ui/ui.py

@@ -1,14 +1,18 @@
 """This file should be imported only and only if you want to run the UI locally."""
+
 import itertools
 import logging
 from pathlib import Path
+import subprocess
 from typing import Any
+import os
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
 import gradio as gr
 from fastapi import FastAPI
 from gradio.themes.utils.colors import slate
-from llama_index.llms import ChatMessage, MessageRole
 
+from llama_index.llms import MessageRole, ChatMessage
 from app._config import settings
 from app.components.embedding.component import EmbeddingComponent
 from app.components.llm.component import LLMComponent
@@ -18,6 +22,7 @@ from app.enums import PROJECT_ROOT_PATH
 from app.server.chat.service import ChatService
 from app.server.ingest.service import IngestService
 from app.ui.schemas import Source
+from app.paths import local_data_path
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +33,9 @@ UI_TAB_TITLE = "Agriculture Chatbot"
 
 SOURCES_SEPARATOR = "\n\n Sources: \n"
 
+model_name = "VietAI/envit5-translation"
+tokenizer = AutoTokenizer.from_pretrained(model_name)  
+model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
 
 class PrivateGptUi:
     def __init__(
@@ -40,11 +48,43 @@ class PrivateGptUi:
 
         # Cache the UI blocks
         self._ui_block = None
-
         # Initialize system prompt
         self._system_prompt = self._get_default_system_prompt()
 
-    def _chat(self, message: str, history: list[list[str]], *_: Any) -> Any:
+    def _chat(
+        self,
+        message: str,
+        history: list[list[str]],
+        upload_button: Any,
+        system_prompt_input: Any,
+        # show_image: bool,
+    ) -> Any:
+        # logger.info(f"Show image = {show_image}")
+        if "#ảnh" in message:
+            message = message.replace("#ảnh","")
+            vi_message = "vi: " + message
+            outputs = model.generate(tokenizer([vi_message], return_tensors="pt", padding=True).input_ids, max_length=512)
+            en_message = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0].replace('en:','')
+            command = f"""
+                cd {local_data_path}
+                clip-retrieval filter --query "{en_message}" --output_folder "retrieved_folder" --indice_folder "index_folder" --num_results 1
+                """
+            logger.info(command)
+            subprocess.run(command, shell=True, check=True)
+
+            folder_path = f"{local_data_path}/retrieved_folder"
+            files = os.listdir(folder_path)
+            # sort images by most lately retrieved. Keep the old images to show them in chat history
+            files.sort(
+                key=lambda x: os.path.getctime(os.path.join(folder_path, x)),
+                reverse=True,
+            )
+            newest_image = files[0]
+            logger.info(f"Retrieve image {newest_image}")
+
+            return (os.path.relpath(f"{folder_path}/{newest_image}", PROJECT_ROOT_PATH),)
+        
+
         def build_history() -> list[ChatMessage]:
             history_messages: list[ChatMessage] = list(
                 itertools.chain(
@@ -53,7 +93,11 @@ class PrivateGptUi:
                             ChatMessage(content=interaction[0], role=MessageRole.USER),
                             ChatMessage(
                                 # Remove from history content the Sources information
-                                content=interaction[1].split(SOURCES_SEPARATOR)[0],
+                                content=(
+                                    "[Image Output]"
+                                    if isinstance(interaction[1], tuple)
+                                    else (interaction[1]).split(SOURCES_SEPARATOR)[0]
+                                ),
                                 role=MessageRole.ASSISTANT,
                             ),
                         ]
@@ -142,6 +186,12 @@ class PrivateGptUi:
 
             with gr.Row(equal_height=False):
                 with gr.Column(scale=3):
+                    # image_checkbox = gr.Checkbox(
+                    #     label="Show Image",
+                    #     info="Do you want to output relevant image?",
+                    #     value=False,
+                    #     interactive=True,
+                    # )
                     upload_button = gr.components.UploadButton(
                         "Upload File(s)",
                         type="filepath",
@@ -172,7 +222,6 @@ class PrivateGptUi:
                         interactive=True,
                         render=False,
                     )
-
                     # On blur, set system prompt to use in queries
                     system_prompt_input.blur(
                         self._set_system_prompt,
@@ -192,7 +241,11 @@ class PrivateGptUi:
                                 AVATAR_BOT,
                             ),
                         ),
-                        additional_inputs=[upload_button, system_prompt_input],
+                        additional_inputs=[
+                            upload_button,
+                            system_prompt_input,
+                            # image_checkbox,
+                        ],
                     )
         return blocks
 

imgs.py

@@ -0,0 +1,29 @@
+from app._config import settings
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+match settings.IMG_DATASET:
+    case "growstuff":
+        global urls, keyword
+        urls = [
+            "https://www.growstuff.org/harvests.json",
+            "https://www.growstuff.org/crops.json",
+            "https://www.growstuff.org/seeds.json"
+        ]
+        keyword = "thumbnail_url"
+
+thumbnail_urls = set()
+
+for url in urls:
+    response = requests.get(url)
+    if response.status_code == 200:
+        data = response.json()['query']
+        thumbnail_urls.update(item.get(keyword) for item in data if item and item.get(keyword))
+    else:
+        logger.info(f"Failed to retrieve data from {url}.")
+thumbnail_urls = list(thumbnail_urls)
+with open(f'{settings.LOCAL_DATA_FOLDER}/myimglist.txt', 'w') as file:
+    for url in thumbnail_urls:
+        file.write(url + '\n')
+    logger.info(f"Retrieved {len(thumbnail_urls)} image urls and written to {settings.LOCAL_DATA_FOLDER}/myimglist.txt")

index.sh

@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+#pull images url from the web
+python imgs.py
+local_data_folder=$(python -c "from app._config import settings; print(settings.LOCAL_DATA_FOLDER)")
+
+cd "$local_data_folder"
+#remove if folders exists
+rm -rf embeddings_folder
+rm -rf image_folder
+rm -rf index_folder
+rm -rf retrieved_folder
+
+echo "download image urls into image folder"
+img2dataset --url_list=myimglist.txt --output_folder=image_folder --thread_count=64 --image_size=256
+
+echo "create embedding folder"
+#change --num_prepro_workers > 0 to enable multiprocessing
+clip-retrieval inference --input_dataset image_folder --output_folder embeddings_folder --enable_text False --num_prepro_workers 0
+
+echo "create indices from embedding folder"
+clip-retrieval index --embeddings_folder embeddings_folder --index_folder index_folder

pyproject.toml

@@ -4,7 +4,10 @@ version = "0.1.0"
 description = ""
 authors = ["PhucVu <[email protected]>"]
 readme = "README.md"
-
+packages = [
+    { include = "app" },
+    { include = "app/**/*.py" },
+]
 [tool.poetry.dependencies]
 python = "^3.10"
 llama-index = "^0.9.22"
@@ -15,15 +18,14 @@ uvicorn = "^0.25.0"
 pydantic = "^2.5.3"
 gradio = "^4.12.0"
 
-# reranker
-torch = {version="^2.3.0", optional=true}
-sentence-transformers = {version="^2.7.0", optional=true}
-
 [tool.poetry.group.local]
 optional = true
 [tool.poetry.group.local.dependencies]
+torch = "1.13.1"
+clip-retrieval = "^2.44.0"
+img2dataset = "^1.44.1"
+ipython = "^8.20.0"
 transformers = "^4.36.2"
-torch = "^2.1.2"
 llama-cpp-python = "^0.2.29"
 
 [build-system]