Update cluster_semantic_chunker.py

cluster_semantic_chunker.py

@@ -11,8 +11,7 @@ from infinity_emb import AsyncEmbeddingEngine
 
 def bert_token_count(tokenizer, text: str)-> int:
     """Returns the number of tokens in a text string."""
-    print(f'NUM TOKENS : {len(tokenizer(text, add_special_tokens=True))}')
-    return len(tokenizer(text, add_special_tokens=True))
+    return len(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text, add_special_tokens=True)))
 
 class ClusterSemanticChunker(BaseChunker):
     def __init__(self, embed_model: AsyncEmbeddingEngine =None, max_chunk_size=400, min_chunk_size=50, length_function=bert_token_count):