prepare pretrain datasets

scripts/prepare_pretrain_datasets.py

@@ -27,9 +27,9 @@ for i, (block_size, subchunk_size) in enumerate([(4097, 4000)]):
         chunk_size=chunk_size, # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
         num_workers=32,
         reorder_files=False,
-        # This is important to inform LitData that we are encoding contiguous 1D array (tokens).
-        # LitData skips storing metadata for each sample e.g all the tokens are concatenated to form one large tensor.
-        item_loader=TokensLoader(block_size=block_size),
+        ## This is important to inform LitData that we are encoding contiguous 1D array (tokens).
+        ## LitData skips storing metadata for each sample e.g all the tokens are concatenated to form one large tensor.
+        # item_loader=TokensLoader(block_size=block_size),
     )
 
 #