Update app.py

app.py

@@ -85,24 +85,32 @@ def load_and_train():
 
     logging.info("Dataset combinado columnas: %s", combined_dataset.column_names)
 
-    # Función de tokenización en RAM
+    # Función para crear un campo 'text' estandarizado
+    def concatenate_text_fields(examples):
+        texts = []
+        for i in range(len(examples['dialog']) if 'dialog' in examples else 0):
+            if 'dialog' in examples and examples['dialog'][i]:
+                texts.append(examples['dialog'][i])
+            elif 'whole_func_string' in examples and examples['whole_func_string'][i]:
+                texts.append(examples['whole_func_string'][i])
+            elif 'func_documentation_string' in examples and examples['func_documentation_string'][i]:
+                texts.append(examples['func_documentation_string'][i])
+            else:
+                texts.append('')
+        examples['text'] = texts
+        return examples
+
+    # Crear el campo 'text'
+    combined_dataset = combined_dataset.map(concatenate_text_fields, batched=True)
+
+    # Función de tokenización basada en el campo 'text'
     def tokenize_function(examples):
-        text = ""
-        if 'dialog' in examples:
-            text = examples['dialog']
-        elif 'docstring' in examples:
-            text = examples['docstring']
-        elif 'code' in examples:
-            text = examples['code']
-        if text:
-            return tokenizer(text, truncation=True, padding='max_length', max_length=512)
-        return {}
-
-    # Tokenizar y mantener todo en RAM
+        return tokenizer(examples['text'], truncation=True, padding='max_length', max_length=512)
+
+    # Tokenizar el dataset
     tokenized_dataset = combined_dataset.map(
         tokenize_function, 
-        batched=True, 
-        cache_dir=cache_dir
+        batched=True
     )
 
     # Configurar argumentos de entrenamiento