data_budget_hours_24.json

{
    "sources": {
        "the_pile": {
            "provider": "local",
            "file_type": "json",
            "files": [
                "/fs/cml-datasets/Pile/train/00.jsonl.zst",
                "/fs/cml-datasets/Pile/train/01.jsonl.zst",
                "/fs/cml-datasets/Pile/train/02.jsonl.zst",
                "/fs/cml-datasets/Pile/train/03.jsonl.zst",
                "/fs/cml-datasets/Pile/train/04.jsonl.zst",
                "/fs/cml-datasets/Pile/train/05.jsonl.zst",
                "/fs/cml-datasets/Pile/train/06.jsonl.zst",
                "/fs/cml-datasets/Pile/train/07.jsonl.zst",
                "/fs/cml-datasets/Pile/train/08.jsonl.zst",
                "/fs/cml-datasets/Pile/train/09.jsonl.zst",
                "/fs/cml-datasets/Pile/train/10.jsonl.zst",
                "/fs/cml-datasets/Pile/train/11.jsonl.zst",
                "/fs/cml-datasets/Pile/train/12.jsonl.zst",
                "/fs/cml-datasets/Pile/train/13.jsonl.zst",
                "/fs/cml-datasets/Pile/train/14.jsonl.zst",
                "/fs/cml-datasets/Pile/train/15.jsonl.zst",
                "/fs/cml-datasets/Pile/train/16.jsonl.zst",
                "/fs/cml-datasets/Pile/train/17.jsonl.zst",
                "/fs/cml-datasets/Pile/train/18.jsonl.zst",
                "/fs/cml-datasets/Pile/train/19.jsonl.zst",
                "/fs/cml-datasets/Pile/train/20.jsonl.zst",
                "/fs/cml-datasets/Pile/train/21.jsonl.zst",
                "/fs/cml-datasets/Pile/train/22.jsonl.zst",
                "/fs/cml-datasets/Pile/train/23.jsonl.zst",
                "/fs/cml-datasets/Pile/train/24.jsonl.zst",
                "/fs/cml-datasets/Pile/train/25.jsonl.zst",
                "/fs/cml-datasets/Pile/train/26.jsonl.zst",
                "/fs/cml-datasets/Pile/train/27.jsonl.zst",
                "/fs/cml-datasets/Pile/train/28.jsonl.zst",
                "/fs/cml-datasets/Pile/train/29.jsonl.zst"
            ],
            "filter": null,
            "split": "train",
            "streaming": true,
            "remove_columns": null,
            "concatenate_successive_entries": 0
        }
    },
    "name": "the_pile",
    "normalizer": {
        "force_lowercase": true,
        "strip_accents": true,
        "force_english_keyboard": true,
        "whitespace_escape": false
    },
    "tokenizer": "WordPiece",
    "vocab_size": 32768,
    "seq_length": 128,
    "include_cls_token_in_corpus": false,
    "include_sep_token_in_corpus": true,
    "use_type_ids": false,
    "max_entries_in_raw_dataset": 8000000.0,
    "max_seq_in_tokenized_dataset": 85000000.0,
    "named_entity_simplification": false,
    "remove_whitespaces": false,
    "remove_trash": true,
    "trash_cutoff": 0.25,
    "deduplicate_entries": false,
    "deduplication_threshold": 75,
    "ordering": "sentence-length-curriculum"
}