dataset_loader.py

from huggingface_hub import Repository
from huggingface_hub import login
import os
from dotenv import load_dotenv

load_dotenv()


def load_dataset():
    login(token=os.getenv("HUB_TOKEN"))

    repo = Repository(
        local_dir="low_partners_chatbot_data",
        repo_type="dataset",
        clone_from="cjzhi98/low_partner_chatbot_2nd",
        token=True,
    )
    repo.git_pull()