Update app.py

app.py

@@ -11,9 +11,7 @@ import redis
 import uvicorn
 import nltk
 from nltk.stem import WordNetLemmatizer
-from nltk.corpus import wordnet
 from tqdm import tqdm
-from tqdm.keras import TqdmCallback
 from tensorflow.keras import Sequential
 from tensorflow.keras.layers import Dense, Dropout, Input
 from tensorflow.keras.optimizers import SGD
@@ -22,105 +20,92 @@ from fastapi import FastAPI
 from fastapi.responses import HTMLResponse
 from pydantic import BaseModel
 from dotenv import load_dotenv
+from datetime import datetime
+from kareas_nlp import TextProcessor
 
-# Cargar las variables de entorno
 load_dotenv()
 
 app = FastAPI()
 
-# Inicializar el lematizador y Redis
 lemmatizer = WordNetLemmatizer()
 redis_password = os.getenv("REDIS_PASSWORD")
 r = redis.Redis(host=os.getenv("REDIS_HOST"), port=int(os.getenv("REDIS_PORT")), password=redis_password)
 
-# Cargar datos en Redis
-def load_data_to_redis():
-    files_to_load = {
-        'intents.json': 'intents',
-        'classes.pkl': 'classes',
-        'words.pkl': 'words',
-        'chatbot_model.h5': 'chatbot_model'
+def create_intents_json():
+    intents = {
+        "intents": [
+            {
+                "tag": "greeting",
+                "patterns": ["Hola", "¿Cómo estás?", "Buenos días"],
+                "responses": ["¡Hola!", "¿Cómo puedo ayudarte?"],
+                "date": "2021-01-01"
+            },
+            {
+                "tag": "goodbye",
+                "patterns": ["Adiós", "Hasta luego", "Nos vemos"],
+                "responses": ["¡Hasta luego!", "Cuídate!"],
+                "date": "2021-01-01"
+            }
+        ]
     }
 
-    for file_name, redis_key in files_to_load.items():
-        if os.path.exists(file_name) and not r.exists(redis_key):
-            print(f"Cargando {file_name} a Redis...")
-            if file_name.endswith('.json'):
-                with open(file_name) as f:
-                    data = json.load(f)
-                r.set(redis_key, json.dumps(data))
-            elif file_name.endswith('.h5'):
-                with open(file_name, 'rb') as f:
-                    r.set(redis_key, f.read())
-            else:
-                with open(file_name, 'rb') as f:
-                    r.set(redis_key, pickle.dumps(pickle.load(f)))
-
-# Asegurarse de que las carpetas existan
+    with open('intents.json', 'w') as f:
+        json.dump(intents, f, ensure_ascii=False, indent=4)
+
+def load_and_filter_data():
+    with open("intents.json") as file:
+        intents = json.load(file)
+
+    filtered_intents = {
+        "intents": []
+    }
+
+    for intent in intents['intents']:
+        if "date" in intent:
+            intent_date = datetime.strptime(intent["date"], "%Y-%m-%d")
+            if intent_date.year >= 2000 and intent_date <= datetime.now():
+                filtered_intents['intents'].append(intent)
+
+    return filtered_intents
+
 if not os.path.exists('models'):
     os.makedirs('models')
 
-def initialize_redis():
-    global r
-    try:
-        r.ping()
-        print("Conexión a Redis exitosa.")
-        load_data_to_redis()
-    except redis.exceptions.ConnectionError:
-        print("Error al conectar a Redis. Saliendo.")
-        exit(1)
-
 async def train_and_save_model():
     global lemmatizer, r
     while True:
         words, classes, documents = [], [], []
         ignore_words = ['?', '!']
 
-        intents = json.loads(r.get('intents'))
-
-        print("Cargando preguntas de usuario de Redis...")
-        if not r.exists('user_questions_loaded'):
-            user_questions = r.lrange('user_questions', 0, -1)
-            for question in user_questions:
-                question = question.decode('utf-8')
-                try:
-                    existing_tag = r.get(f"tag:{question}").decode('utf-8')
-                    documents.append((nltk.word_tokenize(question), existing_tag))
-                    if existing_tag not in classes:
-                        classes.append(existing_tag)
-                except AttributeError:
-                    documents.append((nltk.word_tokenize(question), "unknown"))
-                    if "unknown" not in classes:
-                        classes.append("unknown")
-            r.set('user_questions_loaded', 1)
-
-        print("Procesando intenciones de Redis...")
+        intents = load_and_filter_data()
+
+        user_questions = r.lrange('user_questions', 0, -1)
+
+        for question in user_questions:
+            question = question.decode('utf-8')
+            processed_words = TextProcessor().process(question)
+            documents.append((processed_words, "user_question"))
+            words.extend(processed_words)
+
         for intent in intents['intents']:
             for pattern in intent['patterns']:
-                w = nltk.word_tokenize(pattern)
-                words.extend(w)
-                documents.append((w, intent['tag']))
+                processed_words = TextProcessor().process(pattern)
+                documents.append((processed_words, intent['tag']))
+                words.extend(processed_words)
                 if intent['tag'] not in classes:
                     classes.append(intent['tag'])
 
-            print(f"Generando sinónimos para la intención '{intent['tag']}'...")
-            with multiprocessing.Pool() as pool:
-                results = []
-                for _ in tqdm(range(100000), desc="Generando sinónimos", leave=False):
-                    if not intent['patterns']:
-                        break
-                    results.append(pool.apply_async(generate_synonym_pattern, (intent['patterns'],)))
-
-                for result in results:
-                    new_pattern = result.get()
-                    if new_pattern:
-                        intent['patterns'].append(new_pattern)
+        for intent in intents['intents']:
+            for pattern in intent['patterns']:
+                synonyms = generate_synonyms(pattern)
+                for synonym in synonyms:
+                    processed_words = TextProcessor().process(synonym)
+                    documents.append((processed_words, intent['tag']))
+                    words.extend(processed_words)
 
-        words = [lemmatizer.lemmatize(w.lower()) for w in words if w not in ignore_words]
         words = sorted(set(words))
         classes = sorted(set(classes))
 
-        print("Creando datos de entrenamiento...")
         training = []
         output_empty = [0] * len(classes)
         for doc in documents:
@@ -134,20 +119,18 @@ async def train_and_save_model():
             training.append([bag, output_row])
 
         if not training:
-            print("Aún no hay datos de entrenamiento. Esperando...")
             await asyncio.sleep(60)
             continue
 
         train_x = np.array([row[0] for row in training])
         train_y = np.array([row[1] for row in training])
 
-        print("Cargando o creando el modelo...")
         if r.exists('chatbot_model'):
             with tempfile.NamedTemporaryFile(delete=False, suffix='.h5') as temp_file:
                 temp_file.write(r.get('chatbot_model'))
                 temp_file_name = temp_file.name
             model = load_model(temp_file_name)
-            os.remove(temp_file_name)
+            os.remove(temp_file.name)
         else:
             input_layer = Input(shape=(len(train_x[0]),))
             layer1 = Dense(128, activation='relu')(input_layer)
@@ -160,10 +143,8 @@ async def train_and_save_model():
             sgd = SGD(learning_rate=0.01, momentum=0.9, nesterov=True)
             model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
 
-        print("Entrenando el modelo...")
-        model.fit(train_x, train_y, epochs=1, batch_size=len(train_x), verbose=0, callbacks=[TqdmCallback(verbose=2)])
+        model.fit(train_x, train_y, epochs=1, batch_size=len(train_x), verbose=0)
 
-        print("Guardando datos en Redis...")
         r.set('words', pickle.dumps(words))
         r.set('classes', pickle.dumps(classes))
 
@@ -173,23 +154,20 @@ async def train_and_save_model():
                 r.set('chatbot_model', f.read())
             os.remove(temp_file.name)
 
-        print("Datos y modelo guardados. Reiniciando entrenamiento...")
-
-def generate_synonym_pattern(patterns):
-    new_pattern = []
-    for word in random.choice(patterns).split():
-        synonyms = wordnet.synsets(word)
-        if synonyms:
-            synonym = random.choice(synonyms[0].lemmas()).name()
-            new_pattern.append(synonym)
-        else:
-            new_pattern.append(word)
-    return " ".join(new_pattern)
-
-def start_training_loop():
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    loop.run_until_complete(train_and_save_model())
+def generate_synonyms(pattern):
+    synonyms = []
+    words = nltk.word_tokenize(pattern)
+    for word in words:
+        synsets = nltk.corpus.wordnet.synsets(word)
+        if synsets:
+            for syn in synsets:
+                for lemma in syn.lemmas():
+                    synonyms.append(lemma.name())
+    return list(set(synonyms))
+
+async def handle_new_message(message: str):
+    r.rpush('user_questions', message)
+    await train_and_save_model()
 
 class ChatMessage(BaseModel):
     message: str
@@ -205,9 +183,7 @@ async def chat(message: ChatMessage):
         model = load_model(temp_file_name)
         os.remove(temp_file.name)
 
-    sentence_words = nltk.word_tokenize(message.message)
-    sentence_words = [lemmatizer.lemmatize(word.lower()) for word in sentence_words]
-
+    sentence_words = TextProcessor().process(message.message)
     bag = [0] * len(words)
     for s in sentence_words:
         for i, w in enumerate(words):
@@ -222,9 +198,7 @@ async def chat(message: ChatMessage):
     for i, p in results:
         return_list.append({"intent": classes[i], "probability": str(p)})
 
-    r.rpush('user_questions', message.message)
-
-    asyncio.create_task(train_and_save_model())
+    await handle_new_message(message.message)
 
     return return_list
 
@@ -326,7 +300,9 @@ async def root():
     return html_code
 
 if __name__ == "__main__":
+    print("Iniciando la aplicación...")
+    create_intents_json()
     initialize_redis()
-    training_process = multiprocessing.Process(target=start_training_loop)
+    training_process = multiprocessing.Process(target=train_and_save_model)
     training_process.start()
     uvicorn.run(app, host="0.0.0.0", port=7860)