huggingface/huggingface_download.py

from huggingface_hub import hf_hub_download
import os

# pip install --no-cache-dir huggingface_hub[hf_transfer]
def single_download(repo, fname, token, endpoint):
    os.environ["TOKIO_WORKER_THREADS"] = "32"
    os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
    file = hf_hub_download(repo_id=repo, filename=fname, token=token, endpoint=endpoint, repo_type="dataset")
    return file

if __name__ == "__main__":
    token = "hf_xxxx"
    repo = "OpenVideo/pexels-raw"
    file = "data/000000.parquet"
    mirror = "https://hf-mirror.com"

    _f = single_download(repo, fname=file, token=token, endpoint=mirror)