Update nlp.py

nlp.py

@@ -3,8 +3,8 @@ import rag_tokenizer
 def tokenize(d, t, eng):
     d["content_with_weight"] = t
     t = re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", t)
-    d["content_ltks"] = tokenize(t)
-    d["content_sm_ltks"] = fine_grained_tokenize(d["content_ltks"])
+    d["content_ltks"] = rag_tokenizer.tokenize(t)
+    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
 
 def is_english(texts):
     eng = 0