Update app.py

app.py

@@ -4,13 +4,16 @@ import numpy as np
 import tensorflow as tf
 from google.cloud import storage
 from tensorflow import keras
-from transformers import TFBertModel, BertTokenizerFast
+from keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional, Input
+from keras.models import Model
 from keras.callbacks import EarlyStopping
 from sklearn.model_selection import train_test_split
 from dotenv import load_dotenv
 import random
 import nltk
 from nltk.corpus import wordnet
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
 
 nltk.download('punkt')
 nltk.download('wordnet')
@@ -174,6 +177,7 @@ def expand_intent(intent):
                 expanded_intents.append(new_intent)
     return expanded_intents
 
+
 num_intents = 100000
 intents = generate_intents(num_intents)
 sentences = generate_sentences(2000)
@@ -208,25 +212,31 @@ labels = [1] * len(intents)
 
 X_train, X_val, y_train, y_val = train_test_split(intents, labels, test_size=0.2, random_state=42)
 
-tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
+tokenizer = Tokenizer(num_words=5000, oov_token="<OOV>")
+tokenizer.fit_on_texts(X_train)
 
-train_encodings = tokenizer(X_train, truncation=True, padding=True, return_tensors="tf")
-val_encodings = tokenizer(X_val, truncation=True, padding=True, return_tensors="tf")
+train_sequences = tokenizer.texts_to_sequences(X_train)
+val_sequences = tokenizer.texts_to_sequences(X_val)
 
-bert_model = TFBertModel.from_pretrained('bert-base-uncased')
+max_length = 128
+train_padded = pad_sequences(train_sequences, maxlen=max_length, padding="post", truncating="post")
+val_padded = pad_sequences(val_sequences, maxlen=max_length, padding="post", truncating="post")
 
-input_ids = keras.layers.Input(shape=(None,), dtype=tf.int32, name="input_ids")
-attention_mask = keras.layers.Input(shape=(None,), dtype=tf.int32, name="attention_mask")
-bert_output = bert_model([input_ids, attention_mask])[1]
-dropout = keras.layers.Dropout(0.1)(bert_output)
-output = keras.layers.Dense(1, activation='sigmoid')(dropout)
+vocab_size = len(tokenizer.word_index) + 1
+embedding_dim = 100
 
-model = keras.Model(inputs=[input_ids, attention_mask], outputs=output)
+input_layer = Input(shape=(max_length,))
+embedding_layer = Embedding(vocab_size, embedding_dim)(input_layer)
+lstm_layer = Bidirectional(LSTM(64))(embedding_layer)
+dropout_layer = Dropout(0.5)(lstm_layer)
+output_layer = Dense(1, activation='sigmoid')(dropout_layer)
 
-model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
+model = Model(inputs=input_layer, outputs=output_layer)
 
+model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)
-history = model.fit(x=train_encodings, y=np.array(y_train), validation_data=(val_encodings, np.array(y_val)), epochs=10, batch_size=16, callbacks=[early_stopping])
+
+model.fit(train_padded, np.array(y_train), validation_data=(val_padded, np.array(y_val)), epochs=10, batch_size=32, callbacks=[early_stopping])
 
 accuracy = history.history['accuracy'][-1]
 print(f"Accuracy: {accuracy}")