update to support lora, added new examples, updated prompt

LLaVA

@@ -1 +1 @@
-Subproject commit a54459c2dd993c4fd2f571cd16bf73cb8dbdcb00
+Subproject commit 30e73a40fe42f392dac3ad9466b3d62e1a40ad07

app.py

@@ -25,7 +25,7 @@ def start_controller():
     return subprocess.Popen(controller_command)
 
 
-def start_worker(model_path: str, bits=16, revision='main', port=21002):
+def start_worker(model_path: str, bits=16, revision='main', model_base = None, port=21002):
     print(f"Starting the model worker for the model {model_path}")
     model_name = model_path.strip("/").split("/")[-1]
     assert bits in [4, 8, 16], "It can be only loaded with 16-bit, 8-bit, and 4-bit."
@@ -49,8 +49,13 @@ def start_worker(model_path: str, bits=16, revision='main', port=21002):
         model_name,
         "--use-flash-attn",
         "--revision",
-        revision
+        revision,
     ]
+    if model_base:
+        worker_command += [
+        "--model-base",
+        model_base
+        ]
     if bits != 16:
         worker_command += [f"--load-{bits}bit"]
     print(worker_command)
@@ -84,6 +89,7 @@ Set the environment variable `model` to change the model:
     print(f"args: {gws.args}")
 
     model_paths = os.getenv("model", "nvandal/LLaVA-Med-v1.5-7b")
+    model_base = os.getenv("model_base", '')
     revisions = os.getenv("revision", "main")
     bits = int(os.getenv("bits", 4))
     concurrency_count = int(os.getenv("concurrency_count", 5))
@@ -93,11 +99,12 @@ Set the environment variable `model` to change the model:
 
     model_paths = model_paths.split(';')
     revisions = revisions.split(';')
+    model_base = model_base.split(';')
     assert(len(model_paths)==len(revisions))
     worker_proc = [None]*len(model_paths)
-    for i, (model_path, revision) in enumerate(zip(model_paths,revisions)):
-        print(model_path, revision)
-        worker_proc[i] = start_worker(model_path, bits=bits, revision=revision, port=str(start_worker_port+i))
+    for i, (model_path, revision, model_base) in enumerate(zip(model_paths,revisions,model_base)):
+        print(model_path, revision, model_base)
+        worker_proc[i] = start_worker(model_path, bits=bits, revision=revision, model_base=model_base, port=str(start_worker_port+i))
 
     # Wait for worker and controller to start
     time.sleep(10)