Update app.py

app.py

@@ -153,7 +153,9 @@ def load_models_and_documents():
             tensor_parallel_size=1,
             max_model_len=131072,
             trust_remote_code=True,
-            enforce_eager=True
+            enforce_eager=True,
+            quantization='gptq',
+            gpu_memory_utilization=0.1
         )
         model = models.VLLM(llm)
 
@@ -387,20 +389,14 @@ if __name__ == '__main__':
             if 'chat_id' not in st.session_state:
                 st.session_state.chat_id = st.selectbox(
                     label='Choose a conversation',
-                    options=[f'{time.time()}'] + list(conversations.keys()),
+                    options=[None] + list(conversations.keys()),
                     format_func=lambda x: conversations.get(x, 'New Chat'),
                     placeholder='_',
                 )
             else:
-                stop_token_ids = [151329, 151336, 151338]
-                sampling_params = SamplingParams(temperature=0.75, max_tokens=35, stop_token_ids=stop_token_ids)
-                prompt = [{"role": "user", "content": f"{}\nExplain the above in one sentence:"}]
-                inputs = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
-                outputs = llm.generate(prompts=inputs, sampling_params=sampling_params)
-
                 st.session_state.chat_id = st.selectbox(
                     label='Choose a conversation',
-                    options=[f'{time.time()}', st.session_state.chat_id] + list(conversations.keys()),
+                    options=[None, st.session_state.chat_id] + list(conversations.keys()),
                     index=1,
                     format_func=lambda x: conversations.get(x, 'New Chat' if x != st.session_state.chat_id else st.session_state.chat_title),
                     placeholder='_',
@@ -420,6 +416,15 @@ if __name__ == '__main__':
                 
         if prompt := st.chat_input("Message Video Game Assistant"):
             if st.session_state.chat_id not in conversations.keys():
+                stop_token_ids = [151329, 151336, 151338]
+                sampling_params = SamplingParams(temperature=0.75, max_tokens=35, stop_token_ids=stop_token_ids)
+                prompt = [{"role": "user", "content": f"{prompt}\nExplain the above in one sentence:"}]
+                inputs = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
+                outputs = llm.generate(prompts=inputs, sampling_params=sampling_params)
+
+                st.session_state.chat_id = outputs[0].outputs[0].text
+                st.session_state.chat_title = f'ChatSession-{st.session_state.chat_id}'
+                
                 conversations[st.session_state.chat_id] = st.session_state.chat_title
                 with lz4.frame.open(conversations_path, mode='wb') as fp:
                     packed_bytes = msgpack.packb(conversations, use_bin_type=True)