Update app.py
Browse files
app.py
CHANGED
|
@@ -42,6 +42,29 @@ lemmatizer = WordNetLemmatizer()
|
|
| 42 |
redis_password = os.getenv("REDIS_PASSWORD")
|
| 43 |
r = redis.Redis(host=os.getenv("REDIS_HOST"), port=int(os.getenv("REDIS_PORT")), password=redis_password)
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
# Ensure directories exist
|
| 46 |
if not os.path.exists('models'):
|
| 47 |
os.makedirs('models')
|
|
@@ -51,6 +74,7 @@ def initialize_redis():
|
|
| 51 |
try:
|
| 52 |
r.ping()
|
| 53 |
print("Redis connection successful.")
|
|
|
|
| 54 |
except redis.exceptions.ConnectionError:
|
| 55 |
print("Error connecting to Redis. Exiting.")
|
| 56 |
exit(1)
|
|
@@ -63,18 +87,7 @@ async def train_and_save_model():
|
|
| 63 |
documents = []
|
| 64 |
ignore_words = ['?', '!']
|
| 65 |
|
| 66 |
-
|
| 67 |
-
if os.path.exists('intents.json'):
|
| 68 |
-
with open('intents.json') as f:
|
| 69 |
-
intents = json.load(f)
|
| 70 |
-
r.set('intents', json.dumps(intents))
|
| 71 |
-
print("Intents loaded from local file and uploaded to Redis.")
|
| 72 |
-
else:
|
| 73 |
-
intents = {"intents": []}
|
| 74 |
-
r.set('intents', json.dumps(intents))
|
| 75 |
-
print("intents.json not found locally, creating empty intents in Redis.")
|
| 76 |
-
else:
|
| 77 |
-
intents = json.loads(r.get('intents'))
|
| 78 |
|
| 79 |
print("Loading user questions from Redis...")
|
| 80 |
if not r.exists('user_questions_loaded'):
|
|
@@ -169,13 +182,14 @@ async def train_and_save_model():
|
|
| 169 |
r.set('words', pickle.dumps(words))
|
| 170 |
r.set('classes', pickle.dumps(classes))
|
| 171 |
|
| 172 |
-
with
|
| 173 |
-
save_model(model,
|
| 174 |
-
|
|
|
|
|
|
|
| 175 |
|
| 176 |
print("Data and model saved. Re-training...")
|
| 177 |
|
| 178 |
-
|
| 179 |
def generate_synonym_pattern(patterns):
|
| 180 |
new_pattern = []
|
| 181 |
for word in random.choice(patterns).split():
|
|
@@ -187,17 +201,14 @@ def generate_synonym_pattern(patterns):
|
|
| 187 |
new_pattern.append(word)
|
| 188 |
return " ".join(new_pattern)
|
| 189 |
|
| 190 |
-
|
| 191 |
def start_training_loop():
|
| 192 |
loop = asyncio.new_event_loop()
|
| 193 |
asyncio.set_event_loop(loop)
|
| 194 |
loop.run_until_complete(train_and_save_model())
|
| 195 |
|
| 196 |
-
|
| 197 |
class ChatMessage(BaseModel):
|
| 198 |
message: str
|
| 199 |
|
| 200 |
-
|
| 201 |
@app.post("/chat")
|
| 202 |
async def chat(message: ChatMessage):
|
| 203 |
words = pickle.loads(r.get('words'))
|
|
@@ -232,13 +243,11 @@ async def chat(message: ChatMessage):
|
|
| 232 |
|
| 233 |
return return_list
|
| 234 |
|
| 235 |
-
|
| 236 |
@app.post("/tag")
|
| 237 |
async def tag_question(question: str, tag: str):
|
| 238 |
r.set(f"tag:{question}", tag)
|
| 239 |
return {"message": "Tag saved"}
|
| 240 |
|
| 241 |
-
|
| 242 |
html_code = """
|
| 243 |
<!DOCTYPE html>
|
| 244 |
<html>
|
|
@@ -336,12 +345,10 @@ html_code = """
|
|
| 336 |
</html>
|
| 337 |
"""
|
| 338 |
|
| 339 |
-
|
| 340 |
@app.get("/", response_class=HTMLResponse)
|
| 341 |
async def root():
|
| 342 |
return html_code
|
| 343 |
|
| 344 |
-
|
| 345 |
if __name__ == "__main__":
|
| 346 |
initialize_redis()
|
| 347 |
training_process = multiprocessing.Process(target=start_training_loop)
|
|
|
|
| 42 |
redis_password = os.getenv("REDIS_PASSWORD")
|
| 43 |
r = redis.Redis(host=os.getenv("REDIS_HOST"), port=int(os.getenv("REDIS_PORT")), password=redis_password)
|
| 44 |
|
| 45 |
+
# Load existing data from files if they exist and upload to Redis
|
| 46 |
+
def load_data_to_redis():
|
| 47 |
+
files_to_load = {
|
| 48 |
+
'intents.json': 'intents',
|
| 49 |
+
'classes.pkl': 'classes',
|
| 50 |
+
'words.pkl': 'words',
|
| 51 |
+
'chatbot_model.h5': 'chatbot_model'
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
for file_name, redis_key in files_to_load.items():
|
| 55 |
+
if os.path.exists(file_name) and not r.exists(redis_key):
|
| 56 |
+
print(f"Loading {file_name} to Redis...")
|
| 57 |
+
if file_name.endswith('.json'):
|
| 58 |
+
with open(file_name) as f:
|
| 59 |
+
data = json.load(f)
|
| 60 |
+
r.set(redis_key, json.dumps(data))
|
| 61 |
+
elif file_name.endswith('.h5'):
|
| 62 |
+
with open(file_name, 'rb') as f:
|
| 63 |
+
r.set(redis_key, f.read())
|
| 64 |
+
else:
|
| 65 |
+
with open(file_name, 'rb') as f:
|
| 66 |
+
r.set(redis_key, pickle.dumps(pickle.load(f)))
|
| 67 |
+
|
| 68 |
# Ensure directories exist
|
| 69 |
if not os.path.exists('models'):
|
| 70 |
os.makedirs('models')
|
|
|
|
| 74 |
try:
|
| 75 |
r.ping()
|
| 76 |
print("Redis connection successful.")
|
| 77 |
+
load_data_to_redis() # Load data on successful connection
|
| 78 |
except redis.exceptions.ConnectionError:
|
| 79 |
print("Error connecting to Redis. Exiting.")
|
| 80 |
exit(1)
|
|
|
|
| 87 |
documents = []
|
| 88 |
ignore_words = ['?', '!']
|
| 89 |
|
| 90 |
+
intents = json.loads(r.get('intents'))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
|
| 92 |
print("Loading user questions from Redis...")
|
| 93 |
if not r.exists('user_questions_loaded'):
|
|
|
|
| 182 |
r.set('words', pickle.dumps(words))
|
| 183 |
r.set('classes', pickle.dumps(classes))
|
| 184 |
|
| 185 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix='.h5') as temp_file:
|
| 186 |
+
save_model(model, temp_file.name)
|
| 187 |
+
with open(temp_file.name, 'rb') as f:
|
| 188 |
+
r.set('chatbot_model', f.read())
|
| 189 |
+
os.remove(temp_file.name)
|
| 190 |
|
| 191 |
print("Data and model saved. Re-training...")
|
| 192 |
|
|
|
|
| 193 |
def generate_synonym_pattern(patterns):
|
| 194 |
new_pattern = []
|
| 195 |
for word in random.choice(patterns).split():
|
|
|
|
| 201 |
new_pattern.append(word)
|
| 202 |
return " ".join(new_pattern)
|
| 203 |
|
|
|
|
| 204 |
def start_training_loop():
|
| 205 |
loop = asyncio.new_event_loop()
|
| 206 |
asyncio.set_event_loop(loop)
|
| 207 |
loop.run_until_complete(train_and_save_model())
|
| 208 |
|
|
|
|
| 209 |
class ChatMessage(BaseModel):
|
| 210 |
message: str
|
| 211 |
|
|
|
|
| 212 |
@app.post("/chat")
|
| 213 |
async def chat(message: ChatMessage):
|
| 214 |
words = pickle.loads(r.get('words'))
|
|
|
|
| 243 |
|
| 244 |
return return_list
|
| 245 |
|
|
|
|
| 246 |
@app.post("/tag")
|
| 247 |
async def tag_question(question: str, tag: str):
|
| 248 |
r.set(f"tag:{question}", tag)
|
| 249 |
return {"message": "Tag saved"}
|
| 250 |
|
|
|
|
| 251 |
html_code = """
|
| 252 |
<!DOCTYPE html>
|
| 253 |
<html>
|
|
|
|
| 345 |
</html>
|
| 346 |
"""
|
| 347 |
|
|
|
|
| 348 |
@app.get("/", response_class=HTMLResponse)
|
| 349 |
async def root():
|
| 350 |
return html_code
|
| 351 |
|
|
|
|
| 352 |
if __name__ == "__main__":
|
| 353 |
initialize_redis()
|
| 354 |
training_process = multiprocessing.Process(target=start_training_loop)
|