app.pyi

'''
Usage:

python -m ferret.serve.gradio_web_server --controller http://localhost:10000 --add_region_feature
'''
import argparse
import datetime
import json
import os
import time

import gradio as gr
import requests

from conversation import (default_conversation, conv_templates,
                                   SeparatorStyle)
from constants import LOGDIR
from utils import (build_logger, server_error_msg,
    violates_moderation, moderation_msg)
import hashlib
# Added 
import re
from copy import deepcopy
from PIL import ImageDraw, ImageFont
from gradio import processing_utils
import numpy as np
import torch
import torch.nn.functional as F
from scipy.ndimage import binary_dilation, binary_erosion
import pdb
from gradio_css import code_highlight_css
import spaces

from inference import inference_and_run

DEFAULT_REGION_REFER_TOKEN = "[region]"
DEFAULT_REGION_FEA_TOKEN = "<region_fea>"


logger = build_logger("gradio_web_server", "gradio_web_server.log")

headers = {"User-Agent": "FERRET Client"}

no_change_btn = gr.Button
enable_btn = gr.Button(interactive=True)
disable_btn = gr.Button(interactive=False)

priority = {
    "vicuna-13b": "aaaaaaa",
    "koala-13b": "aaaaaab",
}

VOCAB_IMAGE_W = 1000  # 224
VOCAB_IMAGE_H = 1000  # 224

def generate_mask_for_feature(coor, raw_w, raw_h, mask=None):
    if mask is not None:
        assert mask.shape[0] == raw_w and mask.shape[1] == raw_h
    coor_mask = torch.zeros((raw_w, raw_h))
    # Assume it samples a point.
    if len(coor) == 2:
        # Define window size
        span = 5
        # Make sure the window does not exceed array bounds
        x_min = max(0, coor[0] - span)
        x_max = min(raw_w, coor[0] + span + 1)
        y_min = max(0, coor[1] - span)
        y_max = min(raw_h, coor[1] + span + 1)
        coor_mask[int(x_min):int(x_max), int(y_min):int(y_max)] = 1
        assert (coor_mask==1).any(), f"coor: {coor}, raw_w: {raw_w}, raw_h: {raw_h}"
    elif len(coor) == 4:
        # Box input or Sketch input.
        coor_mask = torch.zeros((raw_w, raw_h))
        coor_mask[coor[0]:coor[2]+1, coor[1]:coor[3]+1] = 1
        if mask is not None:
            coor_mask = coor_mask * mask
    # coor_mask = torch.from_numpy(coor_mask)
    # pdb.set_trace()
    assert len(coor_mask.nonzero()) != 0
    return coor_mask.tolist()


def draw_box(coor, region_mask, region_ph, img, input_mode):
    colors = ["red"]
    draw = ImageDraw.Draw(img)
    font = ImageFont.truetype("./DejaVuSans.ttf", size=18)
    if input_mode == 'Box':
        draw.rectangle([coor[0], coor[1], coor[2], coor[3]], outline=colors[0], width=4)
        draw.rectangle([coor[0], coor[3] - int(font.size * 1.2), coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[3]], outline=colors[0], fill=colors[0], width=4)
        draw.text([coor[0] + int(font.size * 0.2), coor[3] - int(font.size*1.2)], region_ph, font=font, fill=(255,255,255))
    elif input_mode == 'Point':
        r = 8 
        leftUpPoint = (coor[0]-r, coor[1]-r)
        rightDownPoint = (coor[0]+r, coor[1]+r)
        twoPointList = [leftUpPoint, rightDownPoint]
        draw.ellipse(twoPointList, outline=colors[0], width=4)
        draw.rectangle([coor[0], coor[1], coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[1] + int(font.size * 1.2)], outline=colors[0], fill=colors[0], width=4)
        draw.text([coor[0] + int(font.size * 0.2), coor[1]], region_ph, font=font, fill=(255,255,255))
    elif input_mode == 'Sketch':
        draw.rectangle([coor[0], coor[3] - int(font.size * 1.2), coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[3]], outline=colors[0], fill=colors[0], width=4)
        draw.text([coor[0] + int(font.size * 0.2), coor[3] - int(font.size*1.2)], region_ph, font=font, fill=(255,255,255))
        # Use morphological operations to find the boundary
        mask = np.array(region_mask)
        dilated = binary_dilation(mask, structure=np.ones((3,3)))
        eroded = binary_erosion(mask, structure=np.ones((3,3)))
        boundary = dilated ^ eroded  # XOR operation to find the difference between dilated and eroded mask
        # Loop over the boundary and paint the corresponding pixels
        for i in range(boundary.shape[0]):
            for j in range(boundary.shape[1]):
                if boundary[i, j]:
                    # This is a pixel on the boundary, paint it red
                    draw.point((i, j), fill=colors[0])
    else:
        NotImplementedError(f'Input mode of {input_mode} is not Implemented.')
    return img


def get_conv_log_filename():
    t = datetime.datetime.now()
    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
    return name


# TODO: return model manually just one for now called "jadechoghari/Ferret-UI-Gemma2b"
def get_model_list():
    # ret = requests.post(args.controller_url + "/refresh_all_workers")
    # assert ret.status_code == 200
    # ret = requests.post(args.controller_url + "/list_models")
    # models = ret.json()["models"]
    # models.sort(key=lambda x: priority.get(x, x))
    # logger.info(f"Models: {models}")
    # return models
    models = ["jadechoghari/Ferret-UI-Gemma2b"]
    logger.info(f"Models: {models}")
    return models


get_window_url_params = """
function() {
    const params = new URLSearchParams(window.location.search);
    url_params = Object.fromEntries(params);
    console.log(url_params);
    return url_params;
    }
"""


def load_demo(url_params, request: gr.Request):
    # logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")

    dropdown_update = gr.Dropdown(visible=True)
    if "model" in url_params:
        model = url_params["model"]
        if model in models:
            dropdown_update = gr.Dropdown(
                value=model, visible=True)

    state = default_conversation.copy()
    print("state", state)
    return (state,
            dropdown_update,
            gr.Chatbot(visible=True),
            gr.Textbox(visible=True),
            gr.Button(visible=True),
            gr.Row(visible=True),
            gr.Accordion(visible=True))


def load_demo_refresh_model_list(request: gr.Request):
    # logger.info(f"load_demo. ip: {request.client.host}")
    models = get_model_list()
    state = default_conversation.copy()
    return (state, gr.Dropdown(
               choices=models,
               value=models[0] if len(models) > 0 else ""),
            gr.Chatbot(visible=True),
            gr.Textbox(visible=True),
            gr.Button(visible=True),
            gr.Row(visible=True),
            gr.Accordion(visible=True))


def vote_last_response(state, vote_type, model_selector, request: gr.Request):
    with open(get_conv_log_filename(), "a") as fout:
        data = {
            "tstamp": round(time.time(), 4),
            "type": vote_type,
            "model": model_selector,
            "state": state.dict(),
            "ip": request.client.host,
        }
        fout.write(json.dumps(data) + "\n")


def upvote_last_response(state, model_selector, request: gr.Request):
    vote_last_response(state, "upvote", model_selector, request)
    return ("",) + (disable_btn,) * 3


def downvote_last_response(state, model_selector, request: gr.Request):
    vote_last_response(state, "downvote", model_selector, request)
    return ("",) + (disable_btn,) * 3


def flag_last_response(state, model_selector, request: gr.Request):
    vote_last_response(state, "flag", model_selector, request)
    return ("",) + (disable_btn,) * 3


def regenerate(state, image_process_mode, request: gr.Request):
    state.messages[-1][-1] = None
    prev_human_msg = state.messages[-2]
    if type(prev_human_msg[1]) in (tuple, list):
        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
    state.skip_next = False
    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5


def clear_history(request: gr.Request):
    state = default_conversation.copy()
    return (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5 + \
        (None, {'region_placeholder_tokens':[],'region_coordinates':[],'region_masks':[],'region_masks_in_prompts':[],'masks':[]}, [], None)


def resize_bbox(box, image_w=None, image_h=None, default_wh=VOCAB_IMAGE_W):
    ratio_w = image_w * 1.0 / default_wh
    ratio_h = image_h * 1.0 / default_wh

    new_box = [int(box[0] * ratio_w), int(box[1] * ratio_h), \
               int(box[2] * ratio_w), int(box[3] * ratio_h)]
    return new_box


def show_location(sketch_pad, chatbot):
    image = sketch_pad['image']
    img_w, img_h = image.size
    new_bboxes = []
    old_bboxes = []
    # chatbot[0] is image.
    text = chatbot[1:]
    for round_i in text:
        human_input = round_i[0]
        model_output = round_i[1]
        # TODO: Difference: vocab representation.
        # pattern = r'\[x\d*=(\d+(?:\.\d+)?), y\d*=(\d+(?:\.\d+)?), x\d*=(\d+(?:\.\d+)?), y\d*=(\d+(?:\.\d+)?)\]'
        pattern = r'\[(\d+(?:\.\d+)?), (\d+(?:\.\d+)?), (\d+(?:\.\d+)?), (\d+(?:\.\d+)?)\]'
        matches = re.findall(pattern, model_output)
        for match in matches:
            x1, y1, x2, y2 = map(int, match)
            new_box = resize_bbox([x1, y1, x2, y2], img_w, img_h)
            new_bboxes.append(new_box)
            old_bboxes.append([x1, y1, x2, y2])
        
    set_old_bboxes = sorted(set(map(tuple, old_bboxes)), key=list(map(tuple, old_bboxes)).index)
    list_old_bboxes = list(map(list, set_old_bboxes))

    set_bboxes = sorted(set(map(tuple, new_bboxes)), key=list(map(tuple, new_bboxes)).index)
    list_bboxes = list(map(list, set_bboxes))

    output_image = deepcopy(image)
    draw = ImageDraw.Draw(output_image)
    #TODO: change from local to online path
    font = ImageFont.truetype("./DejaVuSans.ttf", 28)
    for i in range(len(list_bboxes)):
        x1, y1, x2, y2 = list_old_bboxes[i]
        x1_new, y1_new, x2_new, y2_new = list_bboxes[i]
        obj_string = '[obj{}]'.format(i)
        for round_i in text:
            model_output = round_i[1]
            model_output = model_output.replace('[{}, {}, {}, {}]'.format(x1, y1, x2, y2), obj_string)
            round_i[1] = model_output
        draw.rectangle([(x1_new, y1_new), (x2_new, y2_new)], outline="red", width=3)
        draw.text((x1_new+2, y1_new+5), obj_string[1:-1], fill="red", font=font)

    return (output_image, [chatbot[0]] + text, disable_btn)


def add_text(state, text, image_process_mode, original_image, sketch_pad, request: gr.Request):
    print("add text called!")

    
    image = sketch_pad['image']
    print("text", text, "and  : ", len(text))
    print("Image path", original_image)

    if len(text) <= 0 and image is None:
        state.skip_next = True
        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
    if args.moderate:
        flagged = violates_moderation(text)
        if flagged:
            state.skip_next = True
            return (state, state.to_gradio_chatbot(), moderation_msg, None) + (
                no_change_btn,) * 5

    text = text[:1536]  # Hard cut-off
    if original_image is None:
        assert image is not None
        original_image = image.copy()
        print('No location, copy original image in add_text')

    if image is not None:
        if state.first_round:
            text = text[:1200]  # Hard cut-off for images
            if '<image>' not in text:
                # text = '<Image><image></Image>' + text
                text = text + '\n<image>'
            text = (text, original_image, image_process_mode)
            if len(state.get_images(return_pil=True)) > 0:
                new_state = default_conversation.copy()
                new_state.first_round = False
                state=new_state
                print('First round add image finsihed.')

    state.append_message(state.roles[0], text)
    state.append_message(state.roles[1], None)
    state.skip_next = False
    return (state, state.to_gradio_chatbot(), "", original_image) + (disable_btn,) * 5


def post_process_code(code):
    sep = "\n```"
    if sep in code:
        blocks = code.split(sep)
        if len(blocks) % 2 == 1:
            for i in range(1, len(blocks), 2):
                blocks[i] = blocks[i].replace("\\_", "_")
        code = sep.join(blocks)
    return code


def find_indices_in_order(str_list, STR):
    indices = []
    i = 0
    while i < len(STR):
        for element in str_list:
            if STR[i:i+len(element)] == element:
                indices.append(str_list.index(element))
                i += len(element) - 1
                break
        i += 1
    return indices


def format_region_prompt(prompt, refer_input_state):
    # Find regions in prompts and assign corresponding region masks
    refer_input_state['region_masks_in_prompts'] = []
    indices_region_placeholder_in_prompt = find_indices_in_order(refer_input_state['region_placeholder_tokens'], prompt)
    refer_input_state['region_masks_in_prompts'] = [refer_input_state['region_masks'][iii] for iii in indices_region_placeholder_in_prompt]

    # Find regions in prompts and replace with real coordinates and region feature token.
    for region_ph_index, region_ph_i in enumerate(refer_input_state['region_placeholder_tokens']):
        prompt = prompt.replace(region_ph_i, '{} {}'.format(refer_input_state['region_coordinates'][region_ph_index], DEFAULT_REGION_FEA_TOKEN))
    return prompt
    
@spaces.GPU()
def http_bot(state, model_selector, temperature, top_p, max_new_tokens, refer_input_state, request: gr.Request):
# def http_bot(state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request):
    start_tstamp = time.time()
    model_name = model_selector

    if state.skip_next:
        # This generate call is skipped due to invalid inputs
        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
        return

    print("state messages: ", state.messages)
    if len(state.messages) == state.offset + 2:
        # First round of conversation
        # template_name = 'ferret_v1'
        template_name = 'ferret_gemma_instruct'
        # Below is LLaVA's original templates.
        # if "llava" in model_name.lower():
        #     if 'llama-2' in model_name.lower():
        #         template_name = "llava_llama_2"
        #     elif "v1" in model_name.lower():
        #         if 'mmtag' in model_name.lower():
        #             template_name = "v1_mmtag"
        #         elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
        #             template_name = "v1_mmtag"
        #         else:
        #             template_name = "llava_v1"
        #     elif "mpt" in model_name.lower():
        #         template_name = "mpt"
        #     else:
        #         if 'mmtag' in model_name.lower():
        #             template_name = "v0_mmtag"
        #         elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
        #             template_name = "v0_mmtag"
        #         else:
        #             template_name = "llava_v0"
        # elif "mpt" in model_name:
        #     template_name = "mpt_text"
        # elif "llama-2" in model_name:
        #     template_name = "llama_2"
        # else:
        #     template_name = "vicuna_v1"
        new_state = conv_templates[template_name].copy()
        new_state.append_message(new_state.roles[0], state.messages[-2][1])
        new_state.append_message(new_state.roles[1], None)
        state = new_state
        state.first_round = False

    # # Query worker address
    # controller_url = args.controller_url
    # ret = requests.post(controller_url + "/get_worker_address",
    #         json={"model": model_name})
    # worker_addr = ret.json()["address"]
    # logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")

    # No available worker
    # if worker_addr == "":
    #     state.messages[-1][-1] = server_error_msg
    #     yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
    #     return

    # Construct prompt
    prompt = state.get_prompt()
    if args.add_region_feature:
        prompt = format_region_prompt(prompt, refer_input_state)

    all_images = state.get_images(return_pil=True)
    all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
    for image, hash in zip(all_images, all_image_hash):
        t = datetime.datetime.now()
        # fishy can remove it
        filename = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg")
        if not os.path.isfile(filename):
            os.makedirs(os.path.dirname(filename), exist_ok=True)
            image.save(filename)

    # Make requests
    pload = {
        "model": model_name,
        "prompt": prompt,
        "temperature": float(temperature),
        "top_p": float(top_p),
        "max_new_tokens": min(int(max_new_tokens), 1536),
        "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
        "images": f'List of {len(state.get_images())} images: {all_image_hash}',
    }
    logger.info(f"==== request ====\n{pload}")
    if args.add_region_feature:
        pload['region_masks'] = refer_input_state['region_masks_in_prompts']
        logger.info(f"==== add region_masks_in_prompts to request ====\n")

    pload['images'] = state.get_images()
    print(f'Input Prompt: {prompt}')
    print("all_image_hash", all_image_hash)

    state.messages[-1][-1] = "▌"
    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
    
    try:
        # Stream output
        stop = state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2
        #TODO: define inference and run function
        results, extracted_texts = inference_and_run(
        image_path=all_image_hash[0], # double check this
        prompt=prompt,
        model_path=model_name,
        conv_mode="ferret_gemma_instruct",  # Default mode from the original function
        temperature=temperature, 
        top_p=top_p,
        max_new_tokens=max_new_tokens,
        stop=stop    # Assuming we want to process the image
        )

        # response = requests.post(worker_addr + "/worker_generate_stream",
        #     headers=headers, json=pload, stream=True, timeout=10)
        response = extracted_texts
        logger.info(f"This is the respone {response}")

        for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
            if chunk:
                data = json.loads(chunk.decode())
                if data["error_code"] == 0:
                    output = data["text"][len(prompt):].strip()
                    output = post_process_code(output)
                    state.messages[-1][-1] = output + "▌"
                    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
                else:
                    output = data["text"] + f" (error_code: {data['error_code']})"
                    state.messages[-1][-1] = output
                    yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
                    return
                time.sleep(0.03)
    except requests.exceptions.RequestException as e:
        state.messages[-1][-1] = server_error_msg
        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
        return

    state.messages[-1][-1] = state.messages[-1][-1][:-1]
    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5

    finish_tstamp = time.time()
    logger.info(f"{output}")

    with open(get_conv_log_filename(), "a") as fout:
        data = {
            "tstamp": round(finish_tstamp, 4),
            "type": "chat",
            "model": model_name,
            "start": round(start_tstamp, 4),
            "finish": round(start_tstamp, 4),
            "state": state.dict(),
            "images": all_image_hash,
            "ip": request.client.host,
        }
        fout.write(json.dumps(data) + "\n")

title_markdown = ("""
# 🦦 Ferret: Refer and Ground Anything Anywhere at Any Granularity
""")
# [[Project Page]](https://llava-vl.github.io) [[Paper]](https://arxiv.org/abs/2304.08485)

tos_markdown = ("""
### Terms of use
By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
""")


learn_more_markdown = ("""
### License
The service is a research preview intended for non-commercial use only
""")


css = code_highlight_css + """
pre {
    white-space: pre-wrap;       /* Since CSS 2.1 */
    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
    white-space: -pre-wrap;      /* Opera 4-6 */
    white-space: -o-pre-wrap;    /* Opera 7 */
    word-wrap: break-word;       /* Internet Explorer 5.5+ */
}
"""

Instructions = '''
Instructions:
1. Select a 'Referring Input Type'
2. Draw on the image to refer to a region/point.
3. Copy the region id from 'Referring Input Type' to refer to a region in your chat.
'''
from gradio.events import Dependency

class ImageMask(gr.components.Image):
    """
    Sets: source="canvas", tool="sketch"
    """

    is_template = True

    def __init__(self, **kwargs):
        super().__init__(source="upload", tool="sketch", interactive=True, **kwargs)

    def preprocess(self, x):
        return super().preprocess(x)
    from typing import Callable, Literal, Sequence, Any, TYPE_CHECKING
    from gradio.blocks import Block
    if TYPE_CHECKING:
        from gradio.components import Timer
    

def draw(input_mode, input, refer_input_state, refer_text_show, imagebox_refer):
    if type(input) == dict:
        image = deepcopy(input['image'])
        mask = deepcopy(input['mask'])
    else:
        mask = deepcopy(input)

    # W, H -> H, W, 3
    image_new = np.asarray(image)
    img_height = image_new.shape[0]
    img_width = image_new.shape[1]

    # W, H, 4 -> H, W
    mask_new = np.asarray(mask)[:,:,0].copy()
    mask_new = torch.from_numpy(mask_new)
    mask_new = (F.interpolate(mask_new.unsqueeze(0).unsqueeze(0), (img_height, img_width), mode='bilinear') > 0)
    mask_new = mask_new[0, 0].transpose(1, 0).long()

    if len(refer_input_state['masks']) == 0:
        last_mask = torch.zeros_like(mask_new)
    else:
        last_mask = refer_input_state['masks'][-1]

    diff_mask = mask_new - last_mask
    if torch.all(diff_mask == 0):
        print('Init Uploading Images.')
        return (refer_input_state, refer_text_show, image)
    else:
        refer_input_state['masks'].append(mask_new)

    if input_mode == 'Point':
        nonzero_points = diff_mask.nonzero()
        nonzero_points_avg_x = torch.median(nonzero_points[:, 0])
        nonzero_points_avg_y = torch.median(nonzero_points[:, 1])
        sampled_coor = [nonzero_points_avg_x, nonzero_points_avg_y]
        # pdb.set_trace()
        cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height)
    elif input_mode == 'Box' or input_mode == 'Sketch':
        # pdb.set_trace()
        x1x2 = diff_mask.max(1)[0].nonzero()[:, 0]
        y1y2 = diff_mask.max(0)[0].nonzero()[:, 0]
        y1, y2 = y1y2.min(), y1y2.max()
        x1, x2 = x1x2.min(), x1x2.max()
        # pdb.set_trace()
        sampled_coor = [x1, y1, x2, y2]
        if input_mode == 'Box':
            cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height)
        else:
            cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height, mask=diff_mask)
    else:
        raise NotImplementedError(f'Input mode of {input_mode} is not Implemented.')

    # TODO(haoxuan): Hack img_size to be 224 here, need to make it a argument.
    if len(sampled_coor) == 2:
        point_x = int(VOCAB_IMAGE_W * sampled_coor[0] / img_width)
        point_y = int(VOCAB_IMAGE_H * sampled_coor[1] / img_height)
        cur_region_coordinates = f'[{int(point_x)}, {int(point_y)}]'
    elif len(sampled_coor) == 4:
        point_x1 = int(VOCAB_IMAGE_W * sampled_coor[0] / img_width)
        point_y1 = int(VOCAB_IMAGE_H * sampled_coor[1] / img_height)
        point_x2 = int(VOCAB_IMAGE_W * sampled_coor[2] / img_width)
        point_y2 = int(VOCAB_IMAGE_H * sampled_coor[3] / img_height)
        cur_region_coordinates = f'[{int(point_x1)}, {int(point_y1)}, {int(point_x2)}, {int(point_y2)}]'

    cur_region_id = len(refer_input_state['region_placeholder_tokens'])
    cur_region_token = DEFAULT_REGION_REFER_TOKEN.split(']')[0] + str(cur_region_id) + ']'
    refer_input_state['region_placeholder_tokens'].append(cur_region_token)
    refer_input_state['region_coordinates'].append(cur_region_coordinates)
    refer_input_state['region_masks'].append(cur_region_masks)
    assert len(refer_input_state['region_masks']) == len(refer_input_state['region_coordinates']) == len(refer_input_state['region_placeholder_tokens'])
    refer_text_show.append((cur_region_token, ''))

    # Show Parsed Referring.
    imagebox_refer = draw_box(sampled_coor, cur_region_masks, \
                         cur_region_token, imagebox_refer, input_mode)

    return (refer_input_state, refer_text_show, imagebox_refer)

def build_demo(embed_mode):
    textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", visible=False, container=False)
    with gr.Blocks(title="FERRET", theme=gr.themes.Base(), css=css) as demo:
        state = gr.State()

        if not embed_mode:
            gr.Markdown(title_markdown)
            gr.Markdown(Instructions)

        with gr.Row():
            with gr.Column(scale=4):
                with gr.Row(elem_id="model_selector_row"):
                    model_selector = gr.Dropdown(
                        choices=models,
                        value=models[0] if len(models) > 0 else "",
                        interactive=True,
                        show_label=False,
                        container=False)

                original_image = gr.Image(type="pil", visible=False)
                image_process_mode = gr.Radio(
                    ["Raw+Processor", "Crop", "Resize", "Pad"],
                    value="Raw+Processor",
                    label="Preprocess for non-square image",
                    visible=False)

                # Added for any-format input.
                sketch_pad = ImageMask(label="Image & Sketch", type="pil", elem_id="img2text")
                refer_input_mode = gr.Radio(
                    ["Point", "Box", "Sketch"],
                    value="Point",
                    label="Referring Input Type")
                refer_input_state = gr.State({'region_placeholder_tokens':[],
                                              'region_coordinates':[],
                                              'region_masks':[],
                                              'region_masks_in_prompts':[],
                                              'masks':[],
                                              })
                refer_text_show = gr.HighlightedText(value=[], label="Referring Input Cache")

                imagebox_refer = gr.Image(type="pil", label="Parsed Referring Input")
                imagebox_output = gr.Image(type="pil", label='Output Vis')

                cur_dir = os.path.dirname(os.path.abspath(__file__))
                # gr.Examples(examples=[
                #     # [f"{cur_dir}/examples/harry-potter-hogwarts.jpg", "What is in [region0]? And what do people use it for?"],
                #     # [f"{cur_dir}/examples/ingredients.jpg", "What objects are in [region0] and [region1]?"],
                #     # [f"{cur_dir}/examples/extreme_ironing.jpg", "What is unusual about this image? And tell me the coordinates of mentioned objects."],
                #     [f"{cur_dir}/examples/ferret.jpg", "What's the relationship between object [region0] and object [region1]?"],
                #     [f"{cur_dir}/examples/waterview.jpg", "What are the things I should be cautious about when I visit here? Tell me the coordinates in response."],
                #     [f"{cur_dir}/examples/flickr_9472793441.jpg", "Describe the image in details."],
                #     # [f"{cur_dir}/examples/coco_000000281759.jpg", "What are the locations of the woman wearing a blue dress, the woman in flowery top, the girl in purple dress, the girl wearing green shirt?"],
                #     [f"{cur_dir}/examples/room_planning.jpg", "How to improve the design of the given room?"],
                #     [f"{cur_dir}/examples/make_sandwitch.jpg", "How can I make a sandwich with available ingredients?"],
                #     [f"{cur_dir}/examples/bathroom.jpg", "What is unusual about this image?"],
                #     [f"{cur_dir}/examples/kitchen.png", "Is the object a man or a chicken? Explain the reason."],
                # ], inputs=[sketch_pad, textbox])

                with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
                    temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
                    top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
                    max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)

            with gr.Column(scale=5):
                chatbot = gr.Chatbot(elem_id="chatbot", label="FERRET", visible=False).style(height=750)
                with gr.Row():
                    with gr.Column(scale=8):
                        textbox.render()
                    with gr.Column(scale=1, min_width=60):
                        submit_btn = gr.Button(value="Submit", visible=False)
                with gr.Row(visible=False) as button_row:
                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
                    # flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
                    #stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
                    regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
                    clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
                    location_btn = gr.Button(value="🪄 Show location", interactive=False)

        if not embed_mode:
            gr.Markdown(tos_markdown)
            gr.Markdown(learn_more_markdown)
        url_params = gr.JSON(visible=False)

        # Register listeners
        btn_list = [upvote_btn, downvote_btn, location_btn, regenerate_btn, clear_btn]
        upvote_btn.click(upvote_last_response,
            [state, model_selector], [textbox, upvote_btn, downvote_btn, location_btn])
        downvote_btn.click(downvote_last_response,
            [state, model_selector], [textbox, upvote_btn, downvote_btn, location_btn])
        # flag_btn.click(flag_last_response,
        #     [state, model_selector], [textbox, upvote_btn, downvote_btn, flag_btn])
        regenerate_btn.click(regenerate, [state, image_process_mode],
            [state, chatbot, textbox] + btn_list).then(
            http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
            [state, chatbot] + btn_list)
        clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox_output, original_image] + btn_list + \
                        [sketch_pad, refer_input_state, refer_text_show, imagebox_refer])
        location_btn.click(show_location,
            [sketch_pad, chatbot], [imagebox_output, chatbot, location_btn])


        #TODO: fix bug text and image not adding when clicking submit
        textbox.submit(add_text, [state, textbox, image_process_mode, original_image, sketch_pad], [state, chatbot, textbox, original_image] + btn_list
            ).then(http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
                   [state, chatbot] + btn_list)

        submit_btn.click(add_text, [state, textbox, image_process_mode, original_image, sketch_pad], [state, chatbot, textbox, original_image] + btn_list
            ).then(http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
                   [state, chatbot] + btn_list)

            

        sketch_pad.edit(
            draw,
            inputs=[refer_input_mode, sketch_pad, refer_input_state, refer_text_show, imagebox_refer],
            outputs=[refer_input_state, refer_text_show, imagebox_refer],
            queue=True,
        )

        if args.model_list_mode == "once":
            demo.load(load_demo, [url_params], [state, model_selector,
                chatbot, textbox, submit_btn, button_row, parameter_row],
                _js=get_window_url_params)
        elif args.model_list_mode == "reload":
            demo.load(load_demo_refresh_model_list, None, [state, model_selector,
                chatbot, textbox, submit_btn, button_row, parameter_row])
        else:
            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")

    return demo


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", type=str, default="0.0.0.0")
    parser.add_argument("--port", type=int)
    parser.add_argument("--controller-url", type=str, default="http://localhost:21001")
    parser.add_argument("--concurrency-count", type=int, default=8)
    parser.add_argument("--model-list-mode", type=str, default="once",
        choices=["once", "reload"])
    parser.add_argument("--share", action="store_true")
    parser.add_argument("--moderate", action="store_true")
    parser.add_argument("--embed", action="store_true")
    parser.add_argument("--add_region_feature", action="store_true")
    args = parser.parse_args()
    logger.info(f"args: {args}")

    models = get_model_list()

    logger.info(args)
    demo = build_demo(args.embed)
    demo.queue(concurrency_count=args.concurrency_count, status_update_rate=10,
               api_open=False).launch(
        server_name=args.host, server_port=args.port, share=True)