Update app.py
Browse files
app.py
CHANGED
@@ -125,7 +125,7 @@ def run_showui(image, query, session_id):
|
|
125 |
|
126 |
def save_and_upload_data(image_path, query, session_id, is_example_image, votes=None):
|
127 |
"""Save the data to a JSON file and upload to S3."""
|
128 |
-
if is_example_image
|
129 |
return
|
130 |
|
131 |
votes = votes or {"upvotes": 0, "downvotes": 0}
|
@@ -146,7 +146,31 @@ def save_and_upload_data(image_path, query, session_id, is_example_image, votes=
|
|
146 |
|
147 |
return data
|
148 |
|
149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
150 |
examples = [
|
151 |
["./examples/app_store.png", "Download Kindle.", True],
|
152 |
["./examples/ios_setting.png", "Turn off Do not disturb.", True],
|
@@ -159,10 +183,34 @@ examples = [
|
|
159 |
["./examples/safari_google.png", "Click on search bar.", True],
|
160 |
]
|
161 |
|
162 |
-
def
|
163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
164 |
state_image_path = gr.State(value=None)
|
165 |
state_session_id = gr.State(value=None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
166 |
|
167 |
with gr.Row():
|
168 |
with gr.Column(scale=3):
|
@@ -174,58 +222,69 @@ def build_demo():
|
|
174 |
)
|
175 |
submit_btn = gr.Button(value="Submit", variant="primary")
|
176 |
|
177 |
-
# Examples component
|
178 |
gr.Examples(
|
179 |
examples=[[e[0], e[1]] for e in examples],
|
180 |
inputs=[imagebox, textbox],
|
181 |
-
outputs=[
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
# Add a hidden dropdown to pass the `is_example` flag
|
186 |
-
is_example_dropdown = gr.Dropdown(
|
187 |
-
choices=["True", "False"],
|
188 |
-
value="False",
|
189 |
-
visible=False,
|
190 |
-
label="Is Example Image",
|
191 |
-
)
|
192 |
-
|
193 |
-
def set_is_example(query):
|
194 |
-
# Find the example and return its `is_example` flag
|
195 |
-
for _, example_query, is_example in examples:
|
196 |
-
if query.strip() == example_query.strip():
|
197 |
-
return str(is_example) # Return as string for Dropdown compatibility
|
198 |
-
return "False"
|
199 |
-
|
200 |
-
textbox.change(
|
201 |
-
set_is_example,
|
202 |
-
inputs=[textbox],
|
203 |
-
outputs=[is_example_dropdown],
|
204 |
)
|
205 |
|
206 |
with gr.Column(scale=8):
|
207 |
output_img = gr.Image(type="pil", label="Output Image")
|
208 |
output_coords = gr.Textbox(label="Clickable Coordinates")
|
209 |
|
210 |
-
with gr.Row(equal_height=True):
|
|
|
|
|
211 |
clear_btn = gr.Button(value="🗑️ Clear", interactive=True)
|
212 |
|
213 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
214 |
submit_btn.click(
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
)
|
219 |
|
220 |
-
# Clear button logic
|
221 |
clear_btn.click(
|
222 |
-
lambda: (None, None, None, None),
|
223 |
inputs=None,
|
224 |
-
outputs=[imagebox, textbox, output_img, output_coords, state_image_path, state_session_id],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
225 |
)
|
226 |
|
227 |
return demo
|
228 |
|
229 |
if __name__ == "__main__":
|
230 |
-
demo = build_demo()
|
231 |
-
demo.launch(
|
|
|
|
|
|
|
|
|
|
|
|
125 |
|
126 |
def save_and_upload_data(image_path, query, session_id, is_example_image, votes=None):
|
127 |
"""Save the data to a JSON file and upload to S3."""
|
128 |
+
if is_example_image:
|
129 |
return
|
130 |
|
131 |
votes = votes or {"upvotes": 0, "downvotes": 0}
|
|
|
146 |
|
147 |
return data
|
148 |
|
149 |
+
def update_vote(vote_type, session_id, is_example_image):
|
150 |
+
"""Update the vote count and re-upload the JSON file."""
|
151 |
+
if is_example_image:
|
152 |
+
return "Example image used. No vote recorded."
|
153 |
+
|
154 |
+
local_file_name = f"{session_id}.json"
|
155 |
+
|
156 |
+
with open(local_file_name, "r") as f:
|
157 |
+
data = json.load(f)
|
158 |
+
|
159 |
+
if vote_type == "upvote":
|
160 |
+
data["votes"]["upvotes"] += 1
|
161 |
+
elif vote_type == "downvote":
|
162 |
+
data["votes"]["downvotes"] += 1
|
163 |
+
|
164 |
+
with open(local_file_name, "w") as f:
|
165 |
+
json.dump(data, f)
|
166 |
+
|
167 |
+
upload_to_s3(local_file_name, 'altair.storage', object_name=f"ootb/{local_file_name}")
|
168 |
+
|
169 |
+
return f"Your {vote_type} has been recorded. Thank you!"
|
170 |
+
|
171 |
+
with open("./assets/showui.png", "rb") as image_file:
|
172 |
+
base64_image = base64.b64encode(image_file.read()).decode("utf-8")
|
173 |
+
|
174 |
examples = [
|
175 |
["./examples/app_store.png", "Download Kindle.", True],
|
176 |
["./examples/ios_setting.png", "Turn off Do not disturb.", True],
|
|
|
183 |
["./examples/safari_google.png", "Click on search bar.", True],
|
184 |
]
|
185 |
|
186 |
+
def check_if_example(query):
|
187 |
+
"""Check if the query matches an example query."""
|
188 |
+
print(f"Checking query: {query}")
|
189 |
+
for _, example_query, is_example in examples:
|
190 |
+
print(f"Comparing with example: {example_query}")
|
191 |
+
if query.strip() == example_query.strip():
|
192 |
+
print("Match found!")
|
193 |
+
return is_example
|
194 |
+
print("No match found.")
|
195 |
+
return False
|
196 |
+
|
197 |
+
def build_demo(embed_mode, concurrency_count=1):
|
198 |
+
with gr.Blocks(title="ShowUI Demo", theme=gr.themes.Default()) as demo:
|
199 |
state_image_path = gr.State(value=None)
|
200 |
state_session_id = gr.State(value=None)
|
201 |
+
state_is_example_image = gr.State(value=False)
|
202 |
+
|
203 |
+
if not embed_mode:
|
204 |
+
gr.HTML(
|
205 |
+
f"""
|
206 |
+
<div style="text-align: center; margin-bottom: 20px;">
|
207 |
+
<div style="display: flex; justify-content: center;">
|
208 |
+
<img src="data:image/png;base64,{base64_image}" alt="ShowUI" width="320" style="margin-bottom: 10px;"/>
|
209 |
+
</div>
|
210 |
+
<p>ShowUI is a lightweight vision-language-action model for GUI agents.</p>
|
211 |
+
</div>
|
212 |
+
"""
|
213 |
+
)
|
214 |
|
215 |
with gr.Row():
|
216 |
with gr.Column(scale=3):
|
|
|
222 |
)
|
223 |
submit_btn = gr.Button(value="Submit", variant="primary")
|
224 |
|
|
|
225 |
gr.Examples(
|
226 |
examples=[[e[0], e[1]] for e in examples],
|
227 |
inputs=[imagebox, textbox],
|
228 |
+
outputs=[state_is_example_image],
|
229 |
+
fn=lambda img, query: check_if_example(query),
|
230 |
+
examples_per_page=3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
231 |
)
|
232 |
|
233 |
with gr.Column(scale=8):
|
234 |
output_img = gr.Image(type="pil", label="Output Image")
|
235 |
output_coords = gr.Textbox(label="Clickable Coordinates")
|
236 |
|
237 |
+
with gr.Row(elem_id="action-buttons", equal_height=True):
|
238 |
+
upvote_btn = gr.Button(value="Looks good!", variant="secondary")
|
239 |
+
downvote_btn = gr.Button(value="Too bad!", variant="secondary")
|
240 |
clear_btn = gr.Button(value="🗑️ Clear", interactive=True)
|
241 |
|
242 |
+
def on_submit(image, query, is_example_image):
|
243 |
+
if image is None:
|
244 |
+
raise ValueError("No image provided. Please upload an image before submitting.")
|
245 |
+
|
246 |
+
session_id = datetime.now().strftime("%Y%m%d_%H%M%S")
|
247 |
+
|
248 |
+
result_image, click_coords, image_path = run_showui(image, query, session_id)
|
249 |
+
|
250 |
+
save_and_upload_data(image_path, query, session_id, is_example_image)
|
251 |
+
|
252 |
+
return result_image, click_coords, image_path, session_id, is_example_image
|
253 |
+
|
254 |
submit_btn.click(
|
255 |
+
on_submit,
|
256 |
+
[imagebox, textbox, state_is_example_image],
|
257 |
+
[output_img, output_coords, state_image_path, state_session_id, state_is_example_image],
|
258 |
)
|
259 |
|
|
|
260 |
clear_btn.click(
|
261 |
+
lambda: (None, None, None, None, None),
|
262 |
inputs=None,
|
263 |
+
outputs=[imagebox, textbox, output_img, output_coords, state_image_path, state_session_id, state_is_example_image],
|
264 |
+
queue=False
|
265 |
+
)
|
266 |
+
|
267 |
+
upvote_btn.click(
|
268 |
+
lambda session_id, is_example_image: update_vote("upvote", session_id, is_example_image),
|
269 |
+
inputs=[state_session_id, state_is_example_image],
|
270 |
+
outputs=[],
|
271 |
+
queue=False
|
272 |
+
)
|
273 |
+
|
274 |
+
downvote_btn.click(
|
275 |
+
lambda session_id, is_example_image: update_vote("downvote", session_id, is_example_image),
|
276 |
+
inputs=[state_session_id, state_is_example_image],
|
277 |
+
outputs=[],
|
278 |
+
queue=False
|
279 |
)
|
280 |
|
281 |
return demo
|
282 |
|
283 |
if __name__ == "__main__":
|
284 |
+
demo = build_demo(embed_mode=False)
|
285 |
+
demo.queue(api_open=False).launch(
|
286 |
+
server_name="0.0.0.0",
|
287 |
+
server_port=7860,
|
288 |
+
ssr_mode=False,
|
289 |
+
debug=True,
|
290 |
+
)
|