File size: 27,824 Bytes
9afd745 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 |
import time
import re
import json
import os
from datetime import datetime
import gradio as gr
import torch
import modules.shared as shared
from modules import chat, ui as ui_module
from modules.utils import gradio
from modules.text_generation import generate_reply_HF, generate_reply_custom
from .llm_web_search import get_webpage_content, langchain_search_duckduckgo, langchain_search_searxng, Generator
from .langchain_websearch import LangchainCompressor
params = {
"display_name": "LLM Web Search",
"is_tab": True,
"enable": True,
"search results per query": 5,
"langchain similarity score threshold": 0.5,
"instant answers": True,
"regular search results": True,
"search command regex": "",
"default search command regex": r"Search_web\(\"(.*)\"\)",
"open url command regex": "",
"default open url command regex": r"Open_url\(\"(.*)\"\)",
"display search results in chat": True,
"display extracted URL content in chat": True,
"searxng url": "",
"cpu only": True,
"chunk size": 500,
"duckduckgo results per query": 10,
"append current datetime": False,
"default system prompt filename": None,
"force search prefix": "Search_web",
"ensemble weighting": 0.5,
"keyword retriever": "bm25",
"splade batch size": 2,
"chunking method": "character-based",
"chunker breakpoint_threshold_amount": 30
}
custom_system_message_filename = None
extension_path = os.path.dirname(os.path.abspath(__file__))
langchain_compressor = None
update_history = None
force_search = False
def setup():
"""
Is executed when the extension gets imported.
:return:
"""
global params
os.environ["TOKENIZERS_PARALLELISM"] = "true"
os.environ["QDRANT__TELEMETRY_DISABLED"] = "true"
try:
with open(os.path.join(extension_path, "settings.json"), "r") as f:
saved_params = json.load(f)
params.update(saved_params)
save_settings() # add keys of newly added feature to settings.json
except FileNotFoundError:
save_settings()
if not os.path.exists(os.path.join(extension_path, "system_prompts")):
os.makedirs(os.path.join(extension_path, "system_prompts"))
toggle_extension(params["enable"])
def save_settings():
global params
with open(os.path.join(extension_path, "settings.json"), "w") as f:
json.dump(params, f, indent=4)
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return gr.HTML(f'<span style="color:lawngreen"> Settings were saved at {current_datetime}</span>',
visible=True)
def toggle_extension(_enable: bool):
global langchain_compressor, custom_system_message_filename
if _enable:
langchain_compressor = LangchainCompressor(device="cpu" if params["cpu only"] else "cuda",
keyword_retriever=params["keyword retriever"],
model_cache_dir=os.path.join(extension_path, "hf_models"))
compressor_model = langchain_compressor.embeddings.client
compressor_model.to(compressor_model._target_device)
custom_system_message_filename = params.get("default system prompt filename")
else:
if not params["cpu only"] and 'langchain_compressor' in globals(): # free some VRAM
model_attrs = ["embeddings", "splade_doc_model", "splade_query_model"]
for model_attr in model_attrs:
if hasattr(langchain_compressor, model_attr):
model = getattr(langchain_compressor, model_attr)
if hasattr(model, "client"):
model.client.to("cpu")
del model.client
else:
if hasattr(model, "to"):
model.to("cpu")
del model
torch.cuda.empty_cache()
params.update({"enable": _enable})
return _enable
def get_available_system_prompts():
try:
return ["None"] + sorted(os.listdir(os.path.join(extension_path, "system_prompts")))
except FileNotFoundError:
return ["None"]
def load_system_prompt(filename: str or None):
global custom_system_message_filename
if not filename:
return
if filename == "None" or filename == "Select custom system message to load...":
custom_system_message_filename = None
return ""
with open(os.path.join(extension_path, "system_prompts", filename), "r") as f:
prompt_str = f.read()
if params["append current datetime"]:
prompt_str += f"\nDate and time of conversation: {datetime.now().strftime('%A %d %B %Y %H:%M')}"
shared.settings['custom_system_message'] = prompt_str
custom_system_message_filename = filename
return prompt_str
def save_system_prompt(filename, prompt):
if not filename:
return
with open(os.path.join(extension_path, "system_prompts", filename), "w") as f:
f.write(prompt)
return gr.HTML(f'<span style="color:lawngreen"> Saved successfully</span>',
visible=True)
def check_file_exists(filename):
if filename == "":
return gr.HTML("", visible=False)
if os.path.exists(os.path.join(extension_path, "system_prompts", filename)):
return gr.HTML(f'<span style="color:orange"> Warning: Filename already exists</span>', visible=True)
return gr.HTML("", visible=False)
def timeout_save_message():
time.sleep(2)
return gr.HTML("", visible=False)
def deactivate_system_prompt():
shared.settings['custom_system_message'] = None
return "None"
def toggle_forced_search(value):
global force_search
force_search = value
def ui():
"""
Creates custom gradio elements when the UI is launched.
:return:
"""
# Inject custom system message into the main textbox if a default one is set
shared.gradio['custom_system_message'].value = load_system_prompt(custom_system_message_filename)
def update_result_type_setting(choice: str):
if choice == "Instant answers":
params.update({"instant answers": True})
params.update({"regular search results": False})
elif choice == "Regular results":
params.update({"instant answers": False})
params.update({"regular search results": True})
elif choice == "Regular results and instant answers":
params.update({"instant answers": True})
params.update({"regular search results": True})
def update_regex_setting(input_str: str, setting_key: str, error_html_element: gr.component):
if input_str == "":
params.update({setting_key: params[f"default {setting_key}"]})
return {error_html_element: gr.HTML("", visible=False)}
try:
compiled = re.compile(input_str)
if compiled.groups > 1:
raise re.error(f"Only 1 capturing group allowed in regex, but there are {compiled.groups}.")
params.update({setting_key: input_str})
return {error_html_element: gr.HTML("", visible=False)}
except re.error as e:
return {error_html_element: gr.HTML(f'<span style="color:red"> Invalid regex. {str(e).capitalize()}</span>',
visible=True)}
def update_default_custom_system_message(check: bool):
if check:
params.update({"default system prompt filename": custom_system_message_filename})
else:
params.update({"default system prompt filename": None})
with gr.Row():
enable = gr.Checkbox(value=lambda: params['enable'], label='Enable LLM web search')
use_cpu_only = gr.Checkbox(value=lambda: params['cpu only'],
label='Run extension on CPU only '
'(Save settings and restart for the change to take effect)')
with gr.Column():
save_settings_btn = gr.Button("Save settings")
saved_success_elem = gr.HTML("", visible=False)
with gr.Row():
result_radio = gr.Radio(
["Regular results", "Regular results and instant answers"],
label="What kind of search results should be returned?",
value=lambda: "Regular results and instant answers" if
(params["regular search results"] and params["instant answers"]) else "Regular results"
)
with gr.Column():
search_command_regex = gr.Textbox(label="Search command regex string",
placeholder=params["default search command regex"],
value=lambda: params["search command regex"])
search_command_regex_error_label = gr.HTML("", visible=False)
with gr.Column():
open_url_command_regex = gr.Textbox(label="Open URL command regex string",
placeholder=params["default open url command regex"],
value=lambda: params["open url command regex"])
open_url_command_regex_error_label = gr.HTML("", visible=False)
with gr.Column():
show_results = gr.Checkbox(value=lambda: params['display search results in chat'],
label='Display search results in chat')
show_url_content = gr.Checkbox(value=lambda: params['display extracted URL content in chat'],
label='Display extracted URL content in chat')
gr.Markdown(value='---')
with gr.Row():
with gr.Column():
gr.Markdown(value='#### Load custom system message\n'
'Select a saved custom system message from within the system_prompts folder or "None" '
'to clear the selection')
system_prompt = gr.Dropdown(
choices=get_available_system_prompts(), label="Select custom system message",
value=lambda: 'Select custom system message to load...' if custom_system_message_filename is None else
custom_system_message_filename, elem_classes='slim-dropdown')
with gr.Row():
set_system_message_as_default = gr.Checkbox(
value=lambda: custom_system_message_filename == params["default system prompt filename"],
label='Set this custom system message as the default')
refresh_button = ui_module.create_refresh_button(system_prompt, lambda: None,
lambda: {'choices': get_available_system_prompts()},
'refresh-button', interactive=True)
refresh_button.elem_id = "custom-sysprompt-refresh"
delete_button = gr.Button('🗑️', elem_classes='refresh-button', interactive=True)
append_datetime = gr.Checkbox(value=lambda: params['append current datetime'],
label='Append current date and time when loading custom system message')
with gr.Column():
gr.Markdown(value='#### Create custom system message')
system_prompt_text = gr.Textbox(label="Custom system message", lines=3,
value=lambda: load_system_prompt(custom_system_message_filename))
sys_prompt_filename = gr.Text(label="Filename")
sys_prompt_save_button = gr.Button("Save Custom system message")
system_prompt_saved_success_elem = gr.HTML("", visible=False)
gr.Markdown(value='---')
with gr.Accordion("Advanced settings", open=False):
ensemble_weighting = gr.Slider(minimum=0, maximum=1, step=0.05, value=lambda: params["ensemble weighting"],
label="Ensemble Weighting", info="Smaller values = More keyword oriented, "
"Larger values = More focus on semantic similarity")
with gr.Row():
keyword_retriever = gr.Radio([("Okapi BM25", "bm25"),("SPLADE", "splade")], label="Sparse keyword retriever",
info="For change to take effect, toggle the extension off and on again",
value=lambda: params["keyword retriever"])
splade_batch_size = gr.Slider(minimum=2, maximum=256, step=2, value=lambda: params["splade batch size"],
label="SPLADE batch size",
info="Smaller values = Slower retrieval (but lower VRAM usage), "
"Larger values = Faster retrieval (but higher VRAM usage). "
"A good trade-off seems to be setting it = 8",
precision=0)
with gr.Row():
chunker = gr.Radio([("Character-based", "character-based"),
("Semantic", "semantic")], label="Chunking method",
value=lambda: params["chunking method"])
chunker_breakpoint_threshold_amount = gr.Slider(minimum=1, maximum=100, step=1,
value=lambda: params["chunker breakpoint_threshold_amount"],
label="Semantic chunking: sentence split threshold (%)",
info="Defines how different two consecutive sentences have"
" to be for them to be split into separate chunks",
precision=0)
gr.Markdown("**Note: Changing the following might result in DuckDuckGo rate limiting or the LM being overwhelmed**")
num_search_results = gr.Number(label="Max. search results to return per query", minimum=1, maximum=100,
value=lambda: params["search results per query"], precision=0)
num_process_search_results = gr.Number(label="Number of search results to process per query", minimum=1,
maximum=100, value=lambda: params["duckduckgo results per query"],
precision=0)
langchain_similarity_threshold = gr.Number(label="Langchain Similarity Score Threshold", minimum=0., maximum=1.,
value=lambda: params["langchain similarity score threshold"])
chunk_size = gr.Number(label="Max. chunk size", info="The maximal size of the individual chunks that each webpage will"
" be split into, in characters", minimum=2, maximum=10000,
value=lambda: params["chunk size"], precision=0)
with gr.Row():
searxng_url = gr.Textbox(label="SearXNG URL",
value=lambda: params["searxng url"])
# Event functions to update the parameters in the backend
enable.input(toggle_extension, enable, enable)
use_cpu_only.change(lambda x: params.update({"cpu only": x}), use_cpu_only, None)
save_settings_btn.click(save_settings, None, [saved_success_elem])
ensemble_weighting.change(lambda x: params.update({"ensemble weighting": x}), ensemble_weighting, None)
keyword_retriever.change(lambda x: params.update({"keyword retriever": x}), keyword_retriever, None)
splade_batch_size.change(lambda x: params.update({"splade batch size": x}), splade_batch_size, None)
chunker.change(lambda x: params.update({"chunking method": x}), chunker, None)
chunker_breakpoint_threshold_amount.change(lambda x: params.update({"chunker breakpoint_threshold_amount": x}),
chunker_breakpoint_threshold_amount, None)
num_search_results.change(lambda x: params.update({"search results per query": x}), num_search_results, None)
num_process_search_results.change(lambda x: params.update({"duckduckgo results per query": x}),
num_process_search_results, None)
langchain_similarity_threshold.change(lambda x: params.update({"langchain similarity score threshold": x}),
langchain_similarity_threshold, None)
chunk_size.change(lambda x: params.update({"chunk size": x}), chunk_size, None)
result_radio.change(update_result_type_setting, result_radio, None)
search_command_regex.change(lambda x: update_regex_setting(x, "search command regex",
search_command_regex_error_label),
search_command_regex, search_command_regex_error_label, show_progress="hidden")
open_url_command_regex.change(lambda x: update_regex_setting(x, "open url command regex",
open_url_command_regex_error_label),
open_url_command_regex, open_url_command_regex_error_label, show_progress="hidden")
show_results.change(lambda x: params.update({"display search results in chat": x}), show_results, None)
show_url_content.change(lambda x: params.update({"display extracted URL content in chat": x}), show_url_content,
None)
searxng_url.change(lambda x: params.update({"searxng url": x}), searxng_url, None)
delete_button.click(
lambda x: x, system_prompt, gradio('delete_filename')).then(
lambda: os.path.join(extension_path, "system_prompts", ""), None, gradio('delete_root')).then(
lambda: gr.update(visible=True), None, gradio('file_deleter'))
shared.gradio['delete_confirm'].click(
lambda: "None", None, system_prompt).then(
None, None, None, _js="() => { document.getElementById('custom-sysprompt-refresh').click() }")
system_prompt.change(load_system_prompt, system_prompt, shared.gradio['custom_system_message'])
system_prompt.change(load_system_prompt, system_prompt, system_prompt_text)
# restore checked state if chosen system prompt matches the default
system_prompt.change(lambda x: x == params["default system prompt filename"], system_prompt,
set_system_message_as_default)
sys_prompt_filename.change(check_file_exists, sys_prompt_filename, system_prompt_saved_success_elem)
sys_prompt_save_button.click(save_system_prompt, [sys_prompt_filename, system_prompt_text],
system_prompt_saved_success_elem,
show_progress="hidden").then(timeout_save_message,
None,
system_prompt_saved_success_elem,
_js="() => { document.getElementById('custom-sysprompt-refresh').click() }",
show_progress="hidden").then(lambda: "", None,
sys_prompt_filename,
show_progress="hidden")
append_datetime.change(lambda x: params.update({"append current datetime": x}), append_datetime, None)
# '.input' = only triggers when user changes the value of the component, not a function
set_system_message_as_default.input(update_default_custom_system_message, set_system_message_as_default, None)
# A dummy checkbox to enable the actual "Force web search" checkbox to trigger a gradio event
force_search_checkbox = gr.Checkbox(value=False, visible=False, elem_id="Force-search-checkbox")
force_search_checkbox.change(toggle_forced_search, force_search_checkbox, None)
def custom_generate_reply(question, original_question, seed, state, stopping_strings, is_chat):
"""
Overrides the main text generation function.
:return:
"""
global update_history, langchain_compressor
if shared.model.__class__.__name__ in ['LlamaCppModel', 'RWKVModel', 'ExllamaModel', 'Exllamav2Model',
'CtransformersModel']:
generate_func = generate_reply_custom
else:
generate_func = generate_reply_HF
if not params['enable']:
for reply in generate_func(question, original_question, seed, state, stopping_strings, is_chat=is_chat):
yield reply
return
web_search = False
read_webpage = False
max_search_results = int(params["search results per query"])
instant_answers = params["instant answers"]
# regular_search_results = params["regular search results"]
langchain_compressor.num_results = int(params["duckduckgo results per query"])
langchain_compressor.similarity_threshold = params["langchain similarity score threshold"]
langchain_compressor.chunk_size = params["chunk size"]
langchain_compressor.ensemble_weighting = params["ensemble weighting"]
langchain_compressor.splade_batch_size = params["splade batch size"]
langchain_compressor.chunking_method = params["chunking method"]
langchain_compressor.chunker_breakpoint_threshold_amount = params["chunker breakpoint_threshold_amount"]
search_command_regex = params["search command regex"]
open_url_command_regex = params["open url command regex"]
searxng_url = params["searxng url"]
display_search_results = params["display search results in chat"]
display_webpage_content = params["display extracted URL content in chat"]
if search_command_regex == "":
search_command_regex = params["default search command regex"]
if open_url_command_regex == "":
open_url_command_regex = params["default open url command regex"]
compiled_search_command_regex = re.compile(search_command_regex)
compiled_open_url_command_regex = re.compile(open_url_command_regex)
if force_search:
question += f" {params['force search prefix']}"
reply = None
for reply in generate_func(question, original_question, seed, state, stopping_strings, is_chat=is_chat):
if force_search:
reply = params["force search prefix"] + reply
search_re_match = compiled_search_command_regex.search(reply)
if search_re_match is not None:
yield reply
original_model_reply = reply
web_search = True
search_term = search_re_match.group(1)
print(f"LLM_Web_search | Searching for {search_term}...")
reply += "\n```plaintext"
reply += "\nSearch tool:\n"
if searxng_url == "":
search_generator = Generator(langchain_search_duckduckgo(search_term,
langchain_compressor,
max_search_results,
instant_answers))
else:
search_generator = Generator(langchain_search_searxng(search_term,
searxng_url,
langchain_compressor,
max_search_results))
try:
for status_message in search_generator:
yield original_model_reply + f"\n*{status_message}*"
search_results = search_generator.value
except Exception as exc:
exception_message = str(exc)
reply += f"The search tool encountered an error: {exception_message}"
print(f'LLM_Web_search | {search_term} generated an exception: {exception_message}')
else:
if search_results != "":
reply += search_results
else:
reply += f"\nThe search tool did not return any results."
reply += "```"
if display_search_results:
yield reply
break
open_url_re_match = compiled_open_url_command_regex.search(reply)
if open_url_re_match is not None:
yield reply
original_model_reply = reply
read_webpage = True
url = open_url_re_match.group(1)
print(f"LLM_Web_search | Reading {url}...")
reply += "\n```plaintext"
reply += "\nURL opener tool:\n"
try:
webpage_content = get_webpage_content(url)
except Exception as exc:
reply += f"Couldn't open {url}. Error message: {str(exc)}"
print(f'LLM_Web_search | {url} generated an exception: {str(exc)}')
else:
reply += f"\nText content of {url}:\n"
reply += webpage_content
reply += "```\n"
if display_webpage_content:
yield reply
break
yield reply
if web_search or read_webpage:
display_results = web_search and display_search_results or read_webpage and display_webpage_content
# Add results to context and continue model output
new_question = chat.generate_chat_prompt(f"{question}{reply}", state)
new_reply = ""
for new_reply in generate_func(new_question, new_question, seed, state,
stopping_strings, is_chat=is_chat):
if display_results:
yield f"{reply}\n{new_reply}"
else:
yield f"{original_model_reply}\n{new_reply}"
if not display_results:
update_history = [state["textbox"], f"{reply}\n{new_reply}"]
def output_modifier(string, state, is_chat=False):
"""
Modifies the output string before it is presented in the UI. In chat mode,
it is applied to the bot's reply. Otherwise, it is applied to the entire
output.
:param string:
:param state:
:param is_chat:
:return:
"""
return string
def custom_css():
"""
Returns custom CSS as a string. It is applied whenever the web UI is loaded.
:return:
"""
return ''
def custom_js():
"""
Returns custom javascript as a string. It is applied whenever the web UI is
loaded.
:return:
"""
with open(os.path.join(extension_path, "script.js"), "r") as f:
return f.read()
def chat_input_modifier(text, visible_text, state):
"""
Modifies both the visible and internal inputs in chat mode. Can be used to
hijack the chat input with custom content.
:param text:
:param visible_text:
:param state:
:return:
"""
return text, visible_text
def state_modifier(state):
"""
Modifies the dictionary containing the UI input parameters before it is
used by the text generation functions.
:param state:
:return:
"""
return state
def history_modifier(history):
"""
Modifies the chat history before the text generation in chat mode begins.
:param history:
:return:
"""
global update_history
if update_history:
history["internal"].append(update_history)
update_history = None
return history
|