Upload folder using huggingface_hub

.gitattributes

@@ -37,5 +37,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.txt filter=lfs diff=lfs merge=lfs -text
 03-gene-task/data/dna_protein_full.json filter=lfs diff=lfs merge=lfs -text
 03-gene-task/img/protein-structure-1-2.png filter=lfs diff=lfs merge=lfs -text
+04-gene-sft/data/.ipynb_checkpoints/dna_promoter_300-checkpoint.jsonl filter=lfs diff=lfs merge=lfs -text
+04-gene-sft/data/dna_promoter_300.jsonl filter=lfs diff=lfs merge=lfs -text
 04-gene-sft/sft_data/train_data.json filter=lfs diff=lfs merge=lfs -text
 img/gpt2_bridge.png filter=lfs diff=lfs merge=lfs -text

02-gpt2_bert/.ipynb_checkpoints/1-dna-bpe-checkpoint.ipynb

@@ -284,7 +284,7 @@
    "id": "c24f10dc-1117-4493-9333-5ed6d898f44a",
    "metadata": {},
    "source": [
-    "### 训练DNA BPE分词器\n",
+    "### **训练DNA BPE分词器**\n",
     "\n",
     "以上方法展示了如何对 DNA 和蛋白质序列进行“分词”，以提取有用的特征。选择哪种方法取决于具体的任务需求和数据特性。对于简单的分类或回归任务，K-mer 分解或滑动窗口可能是足够的；而对于更复杂的任务，如序列标注或结构预测，基于词汇表的方法或嵌入表示可能会提供更好的性能。\n",
     "\n",

04-gene-sft/.ipynb_checkpoints/2-gpt2-instruction-ft-checkpoint.ipynb

@@ -127,20 +127,76 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "dc04d5e3-7623-4d59-9f3b-ad03e339db11",
+   "execution_count": 1,
+   "id": "b28e3499-bbff-4548-9f85-2baee088cabf",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dc04d5e3-7623-4d59-9f3b-ad03e339db11",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['sequence', 'label'],\n",
+       "        num_rows: 59195\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from datasets import load_dataset\n",
     "# 1. load ~11k samples from promoters prediction dataset\n",
-    "dataset = load_dataset(\"dnagpt/dna_promoter_300\")\n",
-    "dataset"
+    "dna_dataset = load_dataset(\"dnagpt/dna_promoter_300\")\n",
+    "dna_dataset"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
+   "id": "f7332fa1-3343-4247-b4cc-54733dff6964",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'sequence': 'TAAATACGGAAGTTTATTACTTGAGGAATAGATGGAATCGTCGGGCGTGAGAGATCATAATCGGCTGCTTCTGGGAGCCGCACGTGGGAAAGACTTATCCCCGACGGAGCTGGGACTGGGGCACAAACCGGAAGGAACACATCTGACCGAGAAAGAGACCAAGTGGCTCAGGTAGGACCAAAGCGAGCAAGGCTGCGGGTCCTGTTGCTCTCTGTCCTGTAAATTTAAACGTTACGCCACCTGGTAATGATACCCTCGTCCTCCGAGGCGACAAGTCAGAACTTCCACCAAGGGCATTAC',\n",
+       " 'label': 0}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dna_dataset[\"train\"][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "id": "93d09d8d-f521-49f7-b0e0-7ac089dfbf49",
    "metadata": {},
    "outputs": [],
@@ -169,10 +225,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "9f9c0e5a-6591-47ac-b358-d746a00dfc0a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.', 'input': 'TAAATACGGAAGTTTATTACTTGAGGAATAGATGGAATCGTCGGGCGTGAGAGATCATAATCGGCTGCTTCTGGGAGCCGCACGTGGGAAAGACTTATCCCCGACGGAGCTGGGACTGGGGCACAAACCGGAAGGAACACATCTGACCGAGAAAGAGACCAAGTGGCTCAGGTAGGACCAAAGCGAGCAAGGCTGCGGGTCCTGTTGCTCTCTGTCCTGTAAATTTAAACGTTACGCCACCTGGTAATGATACCCTCGTCCTCCGAGGCGACAAGTCAGAACTTCCACCAAGGGCATTAC', 'output': 'Non-promoter'}\n"
+     ]
+    }
+   ],
    "source": [
     "example = dna_dataset[\"train\"][0]\n",
     "print(build_prompt(example))"
@@ -180,7 +244,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "83070a23-1604-4d28-b371-e01060331ed5",
    "metadata": {},
    "outputs": [],
@@ -199,10 +263,40 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "89fb8ed3-aa58-462f-b2a6-ce445c597a33",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7b84d3a64bf645ada13d0cada2d9f524",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split: 0 examples [00:00, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 59195\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "dna_ft_dataset = load_dataset(\"json\", data_files='data/dna_promoter_300.jsonl')\n",
     "dna_ft_dataset"
@@ -210,10 +304,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "e4f7b75f-6ccb-4fda-8004-40df7d52678f",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 53275\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 5920\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.9, seed=42)\n",
     "data"
@@ -221,7 +335,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "36d9ee0e-8423-4529-aa7e-fda2728fab2f",
    "metadata": {},
    "outputs": [],
@@ -235,13 +349,14 @@
     "from tokenizers import Tokenizer\n",
     "from transformers import GPT2TokenizerFast\n",
     "\n",
+    "#需要使用生物序列+英文的多模态大模型\n",
     "tokenizer = GPT2Tokenizer.from_pretrained(\"dnagpt/gene_eng_gpt2_v0\")\n",
     "tokenizer.pad_token = tokenizer.eos_token"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "id": "871baee0-f06f-4422-a741-af533f7d92e1",
    "metadata": {},
    "outputs": [],
@@ -270,10 +385,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "id": "bca1c275-cc3d-43df-923e-e6604d584226",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "example = data[\"test\"][0]\n",
     "example"
@@ -281,10 +409,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "id": "76f2e027-0a31-4919-bb7e-404c786e1599",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "prompt = build_prompt(example)\n",
     "print(prompt)"
@@ -292,24 +437,80 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
    "id": "932b54ca-7e27-47cd-b67d-7ef8386b6608",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tokens:  Bel ow Ġ is Ġ an Ġ instruc tion Ġ th at Ġ describ es Ġ a Ġ t ask . ĠWrit e Ġ a Ġ respon se Ġ th at Ġ appropri at el y Ġ complet es Ġ the Ġ request . Ċ Ċ # # # ĠIn struc tion : Ċ D eter min e Ġ cor e Ġ promo ter Ġ det ec tion Ġ of Ġ follow ing Ġ d na Ġ sequenc e , ĠTh e Ġ resul t Ġ will Ġ be Ġ on e Ġ of Ġ the Ġ follow ing : ĠN on - promo ter , Ġ promo ter . Ċ Ċ # # # ĠIn put : Ċ CC AGGATGC GC TGACG ACCC GGCTGGC AGGC GGGTCC TCG TGGGCG AGGCG AGGGAGGC GGCG AGAGAGG AGCAATAG TTTCCC ACCGC TCCCTCTC AGGCGC AGGG TCTAG AGAAGC GCG AGGGG ATCTAG AGAAGCC GG AGGGG AGGAAGC GCG AGTCC GCGG CCCGCC CCG TTGCG TCCC ACCCACC GCG TCCCCTCCCC TCCCCTCCC GCTGC GGG AAAAGC GGCCGC GGGCGGC GGCGCCC ACTGTG GGGC GGGC GGAGC GCCGC GGGAGGC GGACG AGATGCG AGCGC GGCCGC Ċ Ċ # # # ĠR esp on se : Ċ promo ter\n"
+     ]
+    }
+   ],
    "source": [
     "print('tokens: ', ' '.join(tokenizer.tokenize(prompt)))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "id": "26671faf-68d0-4a44-978e-e1a24e86c9b1",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "02d2f083e74a45e6ada46b9872822dfc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/53275 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f9ebfc901049446cb70580ee6d90861b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/5920 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['input_ids', 'attention_mask'],\n",
+       "        num_rows: 53275\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['input_ids', 'attention_mask'],\n",
+       "        num_rows: 5920\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "def tokenize_function(example):\n",
     "    prompt =  build_prompt(example)\n",
-    "    result = tokenizer(prompt, padding='max_length', truncation=True, max_length=1024) # max_length=1024\n",
+    "    result = tokenizer(prompt, padding='max_length', truncation=True, max_length=256) # max_length=256\n",
     "    return result\n",
     "\n",
     "\n",
@@ -322,17 +523,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "id": "3d46c8b1-9fb3-431a-87ea-c278468543e7",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['input_ids', 'attention_mask'],\n",
+       "    num_rows: 53275\n",
+       "})"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenized_datasets[\"train\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "id": "26985c81-4335-4ac0-9a5a-84a5b4f2d0e4",
    "metadata": {},
    "outputs": [],
@@ -346,17 +561,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "id": "e18d3095-d6dd-423b-84fb-dca4a629d450",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "08a24dc5ff954b8581d5c2378e0d60d6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/111 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "model = GPT2LMHeadModel.from_pretrained(\"dnagpt/gene_eng_gpt2_v0\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 22,
    "id": "12134cf2-676a-4176-a733-35caab2fd520",
    "metadata": {},
    "outputs": [],
@@ -401,10 +631,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
    "id": "b9a2e2a9-a1ff-44b0-a550-623a16d0d7a2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n",
+      "TATAT\n"
+     ]
+    }
+   ],
    "source": [
     "input_text = format_input(data[\"test\"][0])\n",
     "\n",
@@ -418,7 +679,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 24,
    "id": "63b54fe2-f077-4ca8-974e-1bcc41ce57d6",
    "metadata": {},
    "outputs": [],
@@ -437,10 +698,217 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 25,
    "id": "61df123d-e67d-4548-998a-de1e2781e774",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-01-10 15:41:26,331] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/compiler_compat/ld: cannot find -laio: No such file or directory\n",
+      "collect2: error: ld returned 1 exit status\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libpthread.so.0, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libstdc++.so.6, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libm.so.6, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::runtime_error::~runtime_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__gxx_personality_v0@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::tellp()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::chrono::_V2::steady_clock::now()@GLIBCXX_3.4.19'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_replace_aux(unsigned long, unsigned long, unsigned long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for bool@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::logic_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::~locale()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_end_catch@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ofstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::logic_error::~logic_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for __cxxabiv1::__si_class_type_info@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::_M_cache_locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator new[](unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_leak_hard()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ifstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >::basic_streambuf(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned short@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::resize(unsigned long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for char const*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ctype<char>::_M_widen_init() const@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_invalid_argument(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::operator=(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<wchar_t, std::char_traits<wchar_t> >::_M_cache_locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_decrement(std::_Rb_tree_node_base const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_free_exception@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::notify_one()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::Init::~Init()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_pure_virtual@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::flush()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for __cxxabiv1::__class_type_info@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_rethrow@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_fstream<char, std::char_traits<char> >::~basic_fstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::compare(char const*) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ostringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::locale()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::chrono::_V2::system_clock::now()@GLIBCXX_3.4.19'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ifstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Hash_bytes(void const*, unsigned long, unsigned long)@CXXABI_1.3.5'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<long long>(long long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for char*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_Prime_rehash_policy::_M_need_rehash(unsigned long, unsigned long, unsigned long) const@GLIBCXX_3.4.18'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::out_of_range@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<unsigned long>(unsigned long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_increment(std::_Rb_tree_node_base const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::~ios_base()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::range_error::~range_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__basic_file<char>::~__basic_file()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_guard_acquire@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<bool>(bool)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::overflow_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_fstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::range_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ios<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_filebuf<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator delete[](void*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(unsigned long, char, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_transfer(std::__detail::_List_node_base*, std::__detail::_List_node_base*)@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::replace(unsigned long, unsigned long, char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for std::exception@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_Rep::_M_destroy(std::allocator<wchar_t> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream& std::istream::_M_extract<double>(double&)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_fstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::basic_ifstream(char const*, std::_Ios_Openmode)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator new(unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_istringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned int@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::domain_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::find(char, unsigned long) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::put(char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for int@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_bad_alloc()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_thread_atexit@CXXABI_1.3.7'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_increment(std::_Rb_tree_node_base*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::~basic_ifstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::Init::Init()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::condition_variable()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::basic_filebuf()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::domain_error::~domain_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::cerr@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::find(char const*, unsigned long, unsigned long) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::str() const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::invalid_argument@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for void*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::assign(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_ostringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_rebalance_for_erase(std::_Rb_tree_node_base*, std::_Rb_tree_node_base&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_hook(std::__detail::_List_node_base*)@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_unhook()@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ostringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::_M_sync(char*, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_iostream<char, std::char_traits<char> >::~basic_iostream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_istringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `log2f@GLIBC_2.2.5'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::operator<<(std::basic_streambuf<char, std::char_traits<char> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::exception::~exception()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__basic_file<char>::is_open() const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_istringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::swap(std::string&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<char, std::char_traits<char> >::basic_streambuf(std::basic_streambuf<char, std::char_traits<char> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::init(std::basic_streambuf<char, std::char_traits<char> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_bad_cast()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::clear(std::_Ios_Iostate)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >::operator=(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator delete(void*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::operator<<(int)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_S_empty_rep_storage@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_M_destroy(std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_iostream<wchar_t, std::char_traits<wchar_t> >::~basic_iostream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::runtime_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ofstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_insert_and_rebalance(bool, std::_Rb_tree_node_base*, std::_Rb_tree_node_base*, std::_Rb_tree_node_base&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_stringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_stringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<long>(long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream::get()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::operator<< <std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::out_of_range::~out_of_range()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::length_error::~length_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::invalid_argument::~invalid_argument()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::swap(std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::cout@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<unsigned long long>(unsigned long long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for int*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<void const*>(void const*)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::underflow_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_streambuf<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for std::out_of_range@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_allocate_exception@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ios<wchar_t, std::char_traits<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for void const*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<wchar_t, std::char_traits<wchar_t> >::init(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::reserve(unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_begin_catch@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_Rep::_S_empty_rep_storage@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_leak()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::open(char const*, std::_Ios_Openmode)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_M_sync(wchar_t*, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream::getline(char*, long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_istream<char, std::char_traits<char> >& std::getline<char, std::char_traits<char>, std::allocator<char> >(std::basic_istream<char, std::char_traits<char> >&, std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::~condition_variable()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringbuf<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::insert(unsigned long, char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::assign(char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned char@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::ios_base()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_out_of_range(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::overflow_error::~overflow_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_length_error(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_system_error(int)@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ofstream<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<double>(double)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<char, std::char_traits<char> >::operator=(std::basic_streambuf<char, std::char_traits<char> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for long long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, unsigned long, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_guard_release@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_throw@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::underflow_error::~underflow_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_decrement(std::_Rb_tree_node_base*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::length_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::~basic_filebuf()@GLIBCXX_3.4'\n",
+      "collect2: error: ld returned 1 exit status\n"
+     ]
+    }
+   ],
    "source": [
     "# 初始化Trainer\n",
     "trainer = Trainer(\n",
@@ -454,10 +922,204 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 26,
    "id": "a9cd936a-5ea6-43e3-9848-27080f818606",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='19980' max='19980' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [19980/19980 21:55, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>2.335700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1000</td>\n",
+       "      <td>2.184100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1500</td>\n",
+       "      <td>2.178500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2000</td>\n",
+       "      <td>2.172400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2500</td>\n",
+       "      <td>2.171400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3000</td>\n",
+       "      <td>2.171900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3500</td>\n",
+       "      <td>2.163100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4000</td>\n",
+       "      <td>2.159300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4500</td>\n",
+       "      <td>2.161000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5000</td>\n",
+       "      <td>2.160200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5500</td>\n",
+       "      <td>2.158400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6000</td>\n",
+       "      <td>2.151600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6500</td>\n",
+       "      <td>2.153700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7000</td>\n",
+       "      <td>2.129500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7500</td>\n",
+       "      <td>2.119100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8000</td>\n",
+       "      <td>2.119800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8500</td>\n",
+       "      <td>2.121600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9000</td>\n",
+       "      <td>2.122500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9500</td>\n",
+       "      <td>2.122300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10000</td>\n",
+       "      <td>2.121500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10500</td>\n",
+       "      <td>2.119500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11000</td>\n",
+       "      <td>2.123500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11500</td>\n",
+       "      <td>2.119600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12000</td>\n",
+       "      <td>2.119300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12500</td>\n",
+       "      <td>2.121800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13000</td>\n",
+       "      <td>2.123500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13500</td>\n",
+       "      <td>2.103200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14000</td>\n",
+       "      <td>2.080700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14500</td>\n",
+       "      <td>2.082100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15000</td>\n",
+       "      <td>2.082900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15500</td>\n",
+       "      <td>2.086400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16000</td>\n",
+       "      <td>2.086600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16500</td>\n",
+       "      <td>2.083800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17000</td>\n",
+       "      <td>2.085000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17500</td>\n",
+       "      <td>2.082800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18000</td>\n",
+       "      <td>2.077600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18500</td>\n",
+       "      <td>2.080300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19000</td>\n",
+       "      <td>2.086600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19500</td>\n",
+       "      <td>2.084200</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=19980, training_loss=2.1272921145021977, metrics={'train_runtime': 1315.5944, 'train_samples_per_second': 121.485, 'train_steps_per_second': 15.187, 'total_flos': 2.08804995072e+16, 'train_loss': 2.1272921145021977, 'epoch': 3.0})"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "# 开始训练\n",
     "trainer.train()"
@@ -465,10 +1127,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 27,
    "id": "315aae76-44b4-4513-8139-40ef22934873",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved model to: gpt_ft/final\n"
+     ]
+    }
+   ],
    "source": [
     "save_dir = 'gpt_ft/final'\n",
     "trainer.save_model(save_dir)\n",
@@ -477,7 +1147,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 28,
    "id": "28d2dbbc-02ff-4120-b230-b19905a786cd",
    "metadata": {},
    "outputs": [],
@@ -488,20 +1158,92 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 29,
    "id": "08987c3c-063a-4e9b-9ebb-e637b0b5bccd",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(90000, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=90000, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "finetuned_model"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 30,
    "id": "d75010e8-6d6a-40ef-852e-0d705adc3da8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n",
+      "promoterpromoterpromo\n",
+      "--------------------------\n",
+      "\n",
+      "real answer: \n",
+      "\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "print(\"input (test):\", input_text)\n",
     "\n",
@@ -517,10 +1259,217 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 31,
    "id": "64365e15-510e-4abf-92f5-c78b660b37dc",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n"
+     ]
+    }
+   ],
    "source": [
     "test_data = data[\"test\"].select(range(100))\n",
     "\n",
@@ -543,7 +1492,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 32,
    "id": "a45fb780-fc3f-401c-b6e0-6f7d0c1682de",
    "metadata": {},
    "outputs": [],
@@ -556,19 +1505,128 @@
     "# 将 Dataset 对象导出为 JSON 文件\n",
     "# test_data.to_json(output_file)\n",
     "with open(output_file, \"w\") as file:\n",
-    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing"
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "a83c8881-c763-4bba-8b85-584a6722a38e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "presicion 0.94 same 0.49\n"
+     ]
+    }
+   ],
    "source": [
     "import json\n",
     "\n",
     "\n",
+    "output_file = 'gpt2-small3-1024.json'\n",
     "\n",
     "with open(output_file, \"r\") as file:\n",
     "    test_data = json.load(file)\n",
@@ -580,15 +1638,30 @@
     "    output = item[\"output\"]\n",
     "    #output = \" \".join(tokenizer.tokenize(output))\n",
     "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
     "    if model_response == output: #same it\n",
     "        same_sum = same_sum + 1\n",
     "        \n",
-    "    if model_response.find(output)!=-1: #find it\n",
-    "        right_sum = right_sum + 1\n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
     "\n",
     "\n",
-    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)"
+    "print(\"Accuracy\", right_sum/all_num, \"same\", same_sum/all_num)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8bf88885-fb33-406e-9644-cd174a8a2f28",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/4-deepspeed-intro-checkpoint.ipynb

@@ -10,12 +10,172 @@
   },
   {
    "cell_type": "markdown",
-   "id": "3383c2d7-91a9-4940-b3b2-698fb7d9dbb7",
+   "id": "75b8219d-8069-4b18-96c8-d5024ee049f1",
    "metadata": {},
    "source": [
-    "## 使用gpt2+deepspeed训练"
+    "## 大模型并行训练简介\n",
+    "\n",
+    "大模型的并行训练旨在克服单个 GPU 显存的限制和加速训练过程，通常适用于参数规模较大的模型（如 GPT-3、T5 等）。并行训练主要包括以下几种方法，每种方法适用于不同的场景和模型特性。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 数据并行（Data Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将数据切分成多个小批次，每个 GPU 处理其中一部分。\n",
+    "- 模型副本被复制到每个 GPU。\n",
+    "- 每个 GPU 独立计算梯度，最终通过梯度同步（如 AllReduce 操作）更新参数。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 实现简单，是最常用的并行方法。\n",
+    "  - 对模型大小没有限制。\n",
+    "- **缺点**：\n",
+    "  - 模型副本需要完整加载到每个 GPU，占用显存。\n",
+    "  - 在超大规模模型中，显存压力较大。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 参数规模适中，显存可以容纳整个模型的场景。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 模型并行（Model Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型切分成不同的部分，将不同部分分配到不同的 GPU。\n",
+    "- 前向传播和后向传播时，数据在模型的不同部分之间传递。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 不需要复制整个模型，可以支持超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - GPU 之间通信频繁，可能成为性能瓶颈。\n",
+    "  - 实现复杂，切分模型需要精心设计。\n",
+    "  \n",
+    "#### **适用场景**\n",
+    "- 单个 GPU 无法容纳完整模型参数的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- 将 Transformer 的不同层分配到不同的 GPU。\n",
+    "- 常用工具：DeepSpeed 的 Pipeline Parallelism、NVIDIA Megatron-LM。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 张量并行（Tensor Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型内部的张量（如权重矩阵）切分为多个子张量，并分配到不同 GPU。\n",
+    "- GPU 之间协作完成矩阵计算。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 减少了每个 GPU 的显存占用，同时保持模型整体完整性。\n",
+    "- **缺点**：\n",
+    "  - 实现较复杂，需要优化通信操作。\n",
+    "  - 通信开销较高，适合较大批量的训练。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 参数非常大的模型（如 GPT-3）。\n",
+    "- 需要极致优化显存的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- NVIDIA 的 Megatron-LM 和 Hugging Face Transformers 提供了张量并行的支持。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 管道并行（Pipeline Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型分为不同的部分（通常是按层划分），每部分分配到不同的 GPU。\n",
+    "- 数据按照流水线的方式流经每个 GPU。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 减少每个 GPU 的显存压力。\n",
+    "  - 通过流水线增加计算效率。\n",
+    "- **缺点**：\n",
+    "  - 引入流水线延迟。\n",
+    "  - 实现复杂，需管理数据依赖和同步。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 模型非常深，层数较多的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- DeepSpeed 的 Pipeline Parallelism。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 混合并行（Hybrid Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将数据并行、模型并行、张量并行和管道并行组合使用，充分利用多 GPU 资源。\n",
+    "- 不同的并行方法在不同维度协同工作。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 灵活且适应性强，适合超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - 配置复杂，依赖于框架和训练任务。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 超大规模模型（如 GPT-3 或参数量 >1T）。\n",
+    "- 多机多卡的大型训练环境。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- NVIDIA Megatron-LM 和 DeepSpeed 的混合并行支持。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. ZeRO 优化并行（Zero Redundancy Optimizer）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 通过分片存储模型参数、优化器状态和梯度，显著减少每个 GPU 的显存占用。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 极大降低显存需求。\n",
+    "  - 支持超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - 对 GPU 间通信要求较高。\n",
+    "  - 比数据并行复杂。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 超大模型的高效训练。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- DeepSpeed 提供的 ZeRO Stage 1/2/3。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **方法对比**\n",
+    "\n",
+    "| 并行方法      | 主要优点                      | 主要缺点                      | 适用场景                   |\n",
+    "|---------------|-------------------------------|-------------------------------|---------------------------|\n",
+    "| 数据并行      | 简单高效，易实现              | 模型副本占用大量显存          | 模型规模适中，显存足够     |\n",
+    "| 模型并行      | 支持大模型                    | 通信开销大，切分复杂          | 超大模型，显存有限         |\n",
+    "| 张量并行      | 高效利用显存                  | 实现复杂，通信频繁            | 参数规模极大的模型         |\n",
+    "| 管道并行      | 显存需求降低，适合深模型      | 流水线延迟，数据同步复杂      | 层数多的大型模型           |\n",
+    "| 混合并行      | 灵活适配超大规模模型          | 配置复杂，依赖框架            | 超大规模模型（如 GPT-3）   |\n",
+    "| ZeRO 并行     | 极大节省显存，占用少          | 通信成本高                    | 超大规模模型显存优化       |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "- **中等规模模型**：优先使用 **数据并行**。\n",
+    "- **单卡显存不足**：采用 **模型并行** 或 **张量并行**。\n",
+    "- **超大规模模型**：使用 **混合并行** 或 DeepSpeed 的 **ZeRO 优化**。\n",
+    "\n",
+    "对于现代超大规模模型，通常采用混合并行方法，比如 NVIDIA 的 Megatron-LM 和微软的 DeepSpeed，它们综合了多种并行策略，能够有效利用计算资源并加速训练。如果您有具体的硬件环境或模型需求，可以进一步探讨适合的并行方案！"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06ddaa4d-e04a-41e0-beb5-f04dfaebcd54",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "id": "c0d29667-1e75-46df-8f65-cae27609ee3f",
@@ -169,171 +329,19 @@
    ]
   },
   {
-   "cell_type": "markdown",
-   "id": "75b8219d-8069-4b18-96c8-d5024ee049f1",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5372798-ced3-420c-b853-badd3ff05dc1",
    "metadata": {},
-   "source": [
-    "## 大模型并行方法\n",
-    "\n",
-    "大模型的并行训练旨在克服单个 GPU 显存的限制和加速训练过程，通常适用于参数规模较大的模型（如 GPT-3、T5 等）。并行训练主要包括以下几种方法，每种方法适用于不同的场景和模型特性。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **1. 数据并行（Data Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将数据切分成多个小批次，每个 GPU 处理其中一部分。\n",
-    "- 模型副本被复制到每个 GPU。\n",
-    "- 每个 GPU 独立计算梯度，最终通过梯度同步（如 AllReduce 操作）更新参数。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 实现简单，是最常用的并行方法。\n",
-    "  - 对模型大小没有限制。\n",
-    "- **缺点**：\n",
-    "  - 模型副本需要完整加载到每个 GPU，占用显存。\n",
-    "  - 在超大规模模型中，显存压力较大。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 参数规模适中，显存可以容纳整个模型的场景。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **2. 模型并行（Model Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型切分成不同的部分，将不同部分分配到不同的 GPU。\n",
-    "- 前向传播和后向传播时，数据在模型的不同部分之间传递。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 不需要复制整个模型，可以支持超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - GPU 之间通信频繁，可能成为性能瓶颈。\n",
-    "  - 实现复杂，切分模型需要精心设计。\n",
-    "  \n",
-    "#### **适用场景**\n",
-    "- 单个 GPU 无法容纳完整模型参数的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- 将 Transformer 的不同层分配到不同的 GPU。\n",
-    "- 常用工具：DeepSpeed 的 Pipeline Parallelism、NVIDIA Megatron-LM。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **3. 张量并行（Tensor Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型内部的张量（如权重矩阵）切分为多个子张量，并分配到不同 GPU。\n",
-    "- GPU 之间协作完成矩阵计算。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 减少了每个 GPU 的显存占用，同时保持模型整体完整性。\n",
-    "- **缺点**：\n",
-    "  - 实现较复杂，需要优化通信操作。\n",
-    "  - 通信开销较高，适合较大批量的训练。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 参数非常大的模型（如 GPT-3）。\n",
-    "- 需要极致优化显存的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- NVIDIA 的 Megatron-LM 和 Hugging Face Transformers 提供了张量并行的支持。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **4. 管道并行（Pipeline Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型分为不同的部分（通常是按层划分），每部分分配到不同的 GPU。\n",
-    "- 数据按照流水线的方式流经每个 GPU。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 减少每个 GPU 的显存压力。\n",
-    "  - 通过流水线增加计算效率。\n",
-    "- **缺点**：\n",
-    "  - 引入流水线延迟。\n",
-    "  - 实现复杂，需管理数据依赖和同步。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 模型非常深，层数较多的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- DeepSpeed 的 Pipeline Parallelism。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **5. 混合并行（Hybrid Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将数据并行、模型并行、张量并行和管道并行组合使用，充分利用多 GPU 资源。\n",
-    "- 不同的并行方法在不同维度协同工作。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 灵活且适应性强，适合超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - 配置复杂，依赖于框架和训练任务。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 超大规模模型（如 GPT-3 或参数量 >1T）。\n",
-    "- 多机多卡的大型训练环境。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- NVIDIA Megatron-LM 和 DeepSpeed 的混合并行支持。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **6. ZeRO 优化并行（Zero Redundancy Optimizer）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 通过分片存储模型参数、优化器状态和梯度，显著减少每个 GPU 的显存占用。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 极大降低显存需求。\n",
-    "  - 支持超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - 对 GPU 间通信要求较高。\n",
-    "  - 比数据并行复杂。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 超大模型的高效训练。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- DeepSpeed 提供的 ZeRO Stage 1/2/3。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **方法对比**\n",
-    "\n",
-    "| 并行方法      | 主要优点                      | 主要缺点                      | 适用场景                   |\n",
-    "|---------------|-------------------------------|-------------------------------|---------------------------|\n",
-    "| 数据并行      | 简单高效，易实现              | 模型副本占用大量显存          | 模型规模适中，显存足够     |\n",
-    "| 模型并行      | 支持大模型                    | 通信开销大，切分复杂          | 超大模型，显存有限         |\n",
-    "| 张量并行      | 高效利用显存                  | 实现复杂，通信频繁            | 参数规模极大的模型         |\n",
-    "| 管道并行      | 显存需求降低，适合深模型      | 流水线延迟，数据同步复杂      | 层数多的大型模型           |\n",
-    "| 混合并行      | 灵活适配超大规模模型          | 配置复杂，依赖框架            | 超大规模模型（如 GPT-3）   |\n",
-    "| ZeRO 并行     | 极大节省显存，占用少          | 通信成本高                    | 超大规模模型显存优化       |\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **总结**\n",
-    "- **中等规模模型**：优先使用 **数据并行**。\n",
-    "- **单卡显存不足**：采用 **模型并行** 或 **张量并行**。\n",
-    "- **超大规模模型**：使用 **混合并行** 或 DeepSpeed 的 **ZeRO 优化**。\n",
-    "\n",
-    "对于现代超大规模模型，通常采用混合并行方法，比如 NVIDIA 的 Megatron-LM 和微软的 DeepSpeed，它们综合了多种并行策略，能够有效利用计算资源并加速训练。如果您有具体的硬件环境或模型需求，可以进一步探讨适合的并行方案！"
-   ]
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "markdown",
    "id": "cd848439-bac8-46b2-9a0f-59ae7c343954",
    "metadata": {},
    "source": [
-    "## deepspeed的并行设计\n",
+    "## deepspeed具体设置\n",
     "\n",
     "\n",
     "是的，DeepSpeed 支持多种并行策略，包括 **数据并行**、**模型并行** 和 **张量并行**，并且可以通过其��置文件灵活地设置这些并行模式。\n",
@@ -544,6 +552,22 @@
     "DeepSpeed 的配置高度灵活，可以根据模型大小、显存限制和硬件条件选择适合的并行策略。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a8e6de4c-adc1-4a1b-840a-c8542b4ed783",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3383c2d7-91a9-4940-b3b2-698fb7d9dbb7",
+   "metadata": {},
+   "source": [
+    "## 使用gpt2+deepspeed训练"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "ab2812bc-f743-4f18-b49c-972781484dc6",
@@ -569,14 +593,6 @@
    "metadata": {},
    "outputs": [],
    "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ce701aeb-c8c7-450a-bbf9-b793a19cd0c6",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/5-peft-intro-checkpoint.ipynb

@@ -5,7 +5,15 @@
    "id": "963e9ae0-ac68-44be-8c7d-fb9842784362",
    "metadata": {},
    "source": [
-    "# 4.6 基于llama的基因大模型指令微调"
+    "# 4.5 peft简介"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4288594-c676-4369-aca1-730446f293d7",
+   "metadata": {},
+   "source": [
+    "## peft"
    ]
   },
   {
@@ -139,6 +147,22 @@
     "如果您需要实现高效微调，可以结合 Hugging Face 的 PEFT 库快速上手。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a70b2631-c9b9-49da-96c6-6760c63040ac",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b47ddf3-85c9-4dd8-bbbb-34fc3bd6aa1b",
+   "metadata": {},
+   "source": [
+    "## GPT2使用peft样例"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -366,14 +390,6 @@
     "    print(name)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "37aa6abb-ab1c-4e9c-b968-579dd74044db",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "id": "0add2f79-f35c-4638-80bb-0d8a87a9b6a7",
@@ -502,11 +518,19 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b4a41750-420f-49c4-845d-69db394794f9",
+   "id": "14f20171-0719-4dfa-b888-147b657ebff4",
    "metadata": {},
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "markdown",
+   "id": "b4e7bff2-2a4f-4a1d-9cb1-dd02aead2f85",
+   "metadata": {},
+   "source": [
+    "## LoraConfig具体配置"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "10c99eb9-8007-4297-972e-7be71768c9c3",
@@ -595,62 +619,6 @@
     "\n",
     "---\n",
     "\n",
-    "### **2. 微调哪些参数，冻结哪些参数**\n",
-    "\n",
-    "LoRA 的核心思想是通过 **分解矩阵**，只更新少量参数，而冻结模型的大部分参数。以下是常见设置的说明：\n",
-    "\n",
-    "#### **微调的参数**\n",
-    "- LoRA 通过 `target_modules` 指定的模块，例如：\n",
-    "  - GPT-2 的 `c_attn`（自注意力模块）。\n",
-    "  - BERT 的 `query` 和 `key`。\n",
-    "- 这些模块是模型中对性能贡献最大的部分，通过微调这些模块，任务性能可以显著提升。\n",
-    "\n",
-    "#### **冻结的参数**\n",
-    "- 除了 `target_modules` 中指定的参数外，所有其他模型参数默认冻结，包括：\n",
-    "  - 预训练权重的绝大部分。\n",
-    "  - 偏置参数（如果 `bias=\"none\"`）。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **3. 一般如何设置**\n",
-    "\n",
-    "#### **（1）针对不同任务调整**\n",
-    "- **文本分类任务**：\n",
-    "  - 优先选择自注意力模块（如 `c_attn`）作为 `target_modules`。\n",
-    "  - `r=8` 或 `r=16` 是常见选择，适中计算开销。\n",
-    "  - 设置适当的 dropout（如 `lora_dropout=0.1`）以防止过拟合。\n",
-    "  \n",
-    "- **语言生成任务**：\n",
-    "  - 对 GPT-2 或 GPT-3，选择 `q_proj` 和 `v_proj`（query 和 value 投影模块）。\n",
-    "  - `r=16` 或更高，适应生成任务的高复杂性。\n",
-    "\n",
-    "- **命名实体识别任务**：\n",
-    "  - 优先选择 `q_proj` 和 `k_proj`（query 和 key 模块）。\n",
-    "\n",
-    "#### **（2）参数量与显存的权衡**\n",
-    "- 如果显存有限，减少 `r` 的值。\n",
-    "- 对小型任务，`r=4` 或 `r=8` 通常已经足够。\n",
-    "\n",
-    "#### **（3）偏置设置**\n",
-    "- 偏置参数的影响较小，在大多数情况下，可以选择 `bias=\"none\"` 保持冻结。\n",
-    "- 对非常依赖偏置的任务（如生成风格微调），可以尝试 `bias=\"lora_only\"`。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **4. 示例：如何选择目标模块**\n",
-    "\n",
-    "#### **GPT-2**\n",
-    "对 GPT-2 来说，以下模块通常是微调的目标：\n",
-    "- **`c_attn`**：注意力模块的组合层。\n",
-    "- **`q_proj` 和 `v_proj`**：Query 和 Value 的线性投影。\n",
-    "\n",
-    "#### **BERT**\n",
-    "对 BERT 来说，以下模块通常是微调的目标：\n",
-    "- **`query`**：Attention 的 Query 模块。\n",
-    "- **`key`**：Attention 的 Key 模块。\n",
-    "\n",
-    "---\n",
-    "\n",
     "### **5. 总结建议**\n",
     "- **微调的参数**：优先选择模型中注意力相关模块。\n",
     "- **冻结的参数**：大部分参数默认冻结以节省显存。\n",
@@ -664,22 +632,82 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "execution_count": 1,
+   "id": "bbc080ba-3ee8-4bc6-afd9-2a3241f1bcda",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at dnagpt/dna_gpt2_v0 and are newly initialized: ['score.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f7e72521368341d38a2b11028715a871",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/5920 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "trainable params: 296,448 || all params: 109,180,416 || trainable%: 0.2715\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/peft/tuners/lora/layer.py:1264: UserWarning: fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
    "source": [
     "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer\n",
     "from peft import LoraConfig, get_peft_model, TaskType\n",
     "from datasets import load_dataset\n",
     "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
+    "from transformers import DataCollatorWithPadding\n",
     "\n",
     "# **1. 加载模型和分词器**\n",
-    "model_name = \"gpt2\"  # 基础模型\n",
+    "model_name = \"dnagpt/dna_gpt2_v0\"  # 基础模型\n",
     "num_labels = 2       # 二分类任务\n",
     "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)\n",
     "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
-    "tokenizer.pad_token = tokenizer.eos_token  # 设置 pad_token 为 eos_token\n",
+    "\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "model.config.pad_token_id = tokenizer.pad_token_id\n",
+    "\n",
     "\n",
     "# **2. 定义数据集**\n",
     "# 示例数据集：dna_promoter_300\n",
@@ -693,7 +721,10 @@
     "    )\n",
     "\n",
     "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
-    "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "#tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "\n",
+    "# 4. 创建一个数据收集器，用于动态填充和遮蔽\n",
+    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
     "\n",
     "# **4. 划分数据集**\n",
     "train_dataset = tokenized_datasets[\"train\"]\n",
@@ -711,8 +742,93 @@
     "\n",
     "# 使用 LoRA 包装模型\n",
     "model = get_peft_model(model, lora_config)\n",
-    "model.print_trainable_parameters()  # 打印可训练的参数信息\n",
-    "\n",
+    "model.print_trainable_parameters()  # 打印可训练的参数信息"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "7da39e7f-db92-483c-888d-19707ab35c5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
+      "  warnings.warn(\n",
+      "/tmp/ipykernel_2399/742597822.py:28: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='19980' max='19980' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [19980/19980 10:13, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>Precision</th>\n",
+       "      <th>Recall</th>\n",
+       "      <th>F1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>0.299700</td>\n",
+       "      <td>0.325549</td>\n",
+       "      <td>0.897635</td>\n",
+       "      <td>0.908117</td>\n",
+       "      <td>0.885483</td>\n",
+       "      <td>0.896658</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>0.304000</td>\n",
+       "      <td>0.290004</td>\n",
+       "      <td>0.904899</td>\n",
+       "      <td>0.889069</td>\n",
+       "      <td>0.925901</td>\n",
+       "      <td>0.907111</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>0.310100</td>\n",
+       "      <td>0.289658</td>\n",
+       "      <td>0.906250</td>\n",
+       "      <td>0.892138</td>\n",
+       "      <td>0.924891</td>\n",
+       "      <td>0.908219</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "训练完成，模型已保存至 ./gpt2_lora_text_classification\n"
+     ]
+    }
+   ],
+   "source": [
     "# **6. 计算指标**\n",
     "def compute_metrics(eval_pred):\n",
     "    predictions, labels = eval_pred\n",
@@ -729,7 +845,7 @@
     "    learning_rate=2e-5,                          # 学习率\n",
     "    per_device_train_batch_size=8,               # 每设备的批量大小\n",
     "    per_device_eval_batch_size=8,                # 每设备评估的批量大小\n",
-    "    num_train_epochs=3,                          # 训练轮数\n",
+    "    num_train_epochs=10,                          # 训练轮数\n",
     "    weight_decay=0.01,                           # 权重衰减\n",
     "    logging_dir=\"./logs\",                        # 日志路径\n",
     "    fp16=True,                                   # 启用混合精度训练\n",
@@ -746,6 +862,7 @@
     "    train_dataset=train_dataset,\n",
     "    eval_dataset=test_dataset,\n",
     "    tokenizer=tokenizer,\n",
+    "    data_collator=data_collator,\n",
     "    compute_metrics=compute_metrics,\n",
     ")\n",
     "\n",

04-gene-sft/.ipynb_checkpoints/6-llama-continue-train-checkpoint.ipynb

@@ -330,6 +330,38 @@
     "本节任务是基于llama。训练一个能够处理dna和protein蛋白质数据的基础预训练大模型，数据为第一章中的预训练数据，包括英文数据。"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "aec90d65-ac62-4394-a526-ca62d8bdbad4",
+   "metadata": {},
+   "source": [
+    "## 环境设置\n",
+    "并行环境对transformer、peft等的版本要求比较高，如果版本不匹配可能会出现各种异常问题\n",
+    "之前的课程，都是单GPU运行，一般不存在版本问题，默认安装的都是最新版本。但运行并行环境时，需要确认下版本再运行，本课程运行并行环境如下：\n",
+    "\n",
+    "* Python 3.12.3\n",
+    "* transformers                   4.45.2\n",
+    "* peft                           0.3.0.dev0\n",
+    "* deepspeed                      0.15.2\n",
+    "* accelerate                     1.0.0\n",
+    "\n",
+    "如果不是，可以重新安装即可：\n",
+    "```\n",
+    "pip install transformers==4.45.2 deepspeed==0.15.2 accelerate==1.0.0\n",
+    "\n",
+    "#peft参考使用的是chinese llama的版本，需要git安装\n",
+    "\n",
+    "git clone https://github.com/huggingface/peft.git\n",
+    "\n",
+    "cd peft\n",
+    "\n",
+    "git checkout 13e53fc\n",
+    "\n",
+    "pip install . \n",
+    "```\n",
+    "如果有环境问题，可以查看本目录下的pip_list.txt"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "b1bd33b8-2e05-4b59-9d8f-c48de194cfd6",
@@ -339,14 +371,24 @@
     "\n",
     "```\n",
     "# 复制第一章训练数据,包括dna，protein，还有英文数据，添加英文数据是为了避免遗忘问题\n",
+    "\n",
     "mkdir train_data\n",
     "cp ../01-data_env/data/*.txt train_data/\n",
+    "使用这些数据，6卡4090大概大致需要训练16个小时，autodl也需要近200块钱了。\n",
+    "\n",
+    "建议学习时，可以使用1/10的数据训练：\n",
+    "awk ‘NR%10==1’ dna_1g.txt > dna.txt\n",
+    "rm dna_1g.txt\n",
+    "其他2类数据依次类推\n",
+    "\n",
+    "这样大概需要2到3个小时就能训练完成了\n",
+    "\n",
     "\n",
     "#持续预训练\n",
     "./run_pt.sh\n",
     "\n",
     "#合并模型\n",
-    "./merge_sft_model.sh\n",
+    "./merge_pt_model.sh\n",
     "\n",
     "```"
    ]
@@ -361,7 +403,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "69b3e97f-a801-4264-a651-a854bcfba9c6",
    "metadata": {},
    "outputs": [],
@@ -376,10 +418,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "339435d9-9379-4b30-ae8b-50feee1ba714",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LlamaTokenizer(name_or_path='dnahlm-merge-hf', vocab_size=91643, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '</s>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
+       "\t0: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "\t1: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-merge-hf\")\n",
     "tokenizer.pad_token = tokenizer.eos_token\n",
@@ -388,10 +445,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "d0f154bb-b1ab-4611-a14c-9b403043fd96",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "342e4ab139b64bb78f0429c2f92c8310",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(91643, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=91643, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "model = LlamaForCausalLM.from_pretrained(\"dnahlm-merge-hf\") #continue pretrain\n",
     "model"
@@ -399,10 +507,51 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "792a9f78-1828-4695-9f6e-479a704ea7e8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LlamaConfig {\n",
+       "  \"_name_or_path\": \"dnahlm-merge-hf\",\n",
+       "  \"architectures\": [\n",
+       "    \"LlamaForCausalLM\"\n",
+       "  ],\n",
+       "  \"attention_bias\": false,\n",
+       "  \"attention_dropout\": 0.0,\n",
+       "  \"bos_token_id\": 1,\n",
+       "  \"eos_token_id\": 2,\n",
+       "  \"head_dim\": 128,\n",
+       "  \"hidden_act\": \"silu\",\n",
+       "  \"hidden_size\": 4096,\n",
+       "  \"initializer_range\": 0.02,\n",
+       "  \"intermediate_size\": 11008,\n",
+       "  \"max_position_embeddings\": 2048,\n",
+       "  \"mlp_bias\": false,\n",
+       "  \"model_type\": \"llama\",\n",
+       "  \"num_attention_heads\": 32,\n",
+       "  \"num_hidden_layers\": 32,\n",
+       "  \"num_key_value_heads\": 32,\n",
+       "  \"pad_token_id\": 0,\n",
+       "  \"pretraining_tp\": 1,\n",
+       "  \"rms_norm_eps\": 1e-06,\n",
+       "  \"rope_scaling\": null,\n",
+       "  \"rope_theta\": 10000.0,\n",
+       "  \"tie_word_embeddings\": false,\n",
+       "  \"torch_dtype\": \"float16\",\n",
+       "  \"transformers_version\": \"4.45.2\",\n",
+       "  \"use_cache\": true,\n",
+       "  \"vocab_size\": 91643\n",
+       "}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from transformers import AutoConfig\n",
     "# 加载配置\n",
@@ -412,10 +561,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "49021c65-54bb-4a97-a96d-b030cc3dcd13",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Test text:\n",
+      " GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
+      "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+      "The primary use of LLaMA is research on large language models, including\n",
+      "Tokenized by DNA-LLaMA tokenizer:['▁GC', 'TGA', 'CT', 'C', 'TGCC', 'AGGATGG', 'AATG', 'AAATT', 'AGGTTG', 'TTTTAATT', 'ATAATGTAA', 'AGTCAG', 'TTCTAG', 'TCAG', 'ACATAG', 'TC', 'ACATAGG', 'CA', 'AGTAAGGG', 'AAC', 'CT', 'AAAATTGC', 'TTGG', 'AAT', ',', '<0x0A>', 'KCG', 'FVGP', 'MVHL', 'KV', 'HLE', 'ADV', 'ASSC', 'RSAV', 'I', 'YL', 'TSEE', 'P', 'FEG', 'VLGL', 'RLK', 'EGI', 'AI', 'TGC', 'W', 'PRW', 'P', 'DEM', 'DER', 'SAV', 'W', 'RVE', 'PY', 'TRH', 'FG', 'RVLY', 'SFGV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n"
+     ]
+    }
+   ],
    "source": [
     "text='''GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
     "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
@@ -426,10 +587,45 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "ebf869c8-866d-4770-8f64-79d671f88663",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e497889a1c3c484cb57c4b6fd93b45ab",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some parameters are on the meta device because they were offloaded to the cpu.\n",
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'The key to life is to accept the fact that you are going to die. The key to'}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "import torch\n",
     "from transformers import pipeline\n",
@@ -448,23 +644,53 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "40a22c70-f1c4-4cd5-a118-2f5db40790e6",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTCTCCTCCTCCTCCTC'}]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "pipe(\"GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCT\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "aec95d0a-4269-4540-bf14-4ce157b9a194",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKETLK'}]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV\")"
+    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLK\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1cfab60-2820-4885-8961-0290c49dfbec",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/7-llama-instruction-ft-checkpoint.ipynb

@@ -184,11 +184,21 @@
     "指令微调通过在特定格式的数据集上进一步训练大模型，使其能够更好地理解和执行用户的自然语言指令。这种方法适合多任务场景，并能提升模型的交互能力和领域适应性。借助高质量的指令数据集和高效的微调技术，大模型在实际应用中的表现可以得到显著提升。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e77f8b39-e75a-4014-a98a-bde5b2534bf1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "id": "7be8b814-42f6-4fb6-bf4b-ae23292030f6",
    "metadata": {},
-   "source": []
+   "source": [
+    "## 持续预训练 VS 指令微调"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -293,6 +303,8 @@
     "#微调\n",
     "./run_sft.sh\n",
     "\n",
+    "运行时间约3小时\n",
+    "\n",
     "#合并模型\n",
     "./merge_sft_model.sh\n",
     "\n",
@@ -325,17 +337,9 @@
     "        os.environ[var] = value"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
-   "metadata": {},
-   "source": [
-    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "054a2956-9045-4ad5-a878-1bfc84ad4ed8",
    "metadata": {},
    "outputs": [],
@@ -350,10 +354,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "63c8bf16-9576-41bc-b27c-c92ba4289cf4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 19839\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from datasets import load_dataset\n",
     "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
@@ -362,10 +382,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "95928da3-ca64-4a17-80f4-945da395702c",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 1983\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 17856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
     "data"
@@ -373,7 +413,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "a3e65bcd-85ce-4261-8ba6-7665c4ec60e2",
    "metadata": {},
    "outputs": [],
@@ -384,10 +424,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "3d3fe49b-f48f-42b2-bc97-028e443111e4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4f060ff2029447b9bad5e2b2e40b7133",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(91644, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=91644, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
     "model"
@@ -395,7 +486,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "c54df9fe-86c4-4963-b313-b438894bf9dd",
    "metadata": {},
    "outputs": [],
@@ -424,10 +515,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "ee540cfb-1f6e-4e02-a3bc-c814e43685cb",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "example = data[\"test\"][0]\n",
     "example"
@@ -435,10 +539,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "7ee35528-7b3f-4e60-b88b-1bc3e950012b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "prompt = build_prompt(example)\n",
     "print(prompt)"
@@ -446,17 +567,116 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "8aa6f38f-3bcc-4566-8a66-a541db91e031",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['▁Below',\n",
+       " '▁is',\n",
+       " '▁an',\n",
+       " '▁instruction',\n",
+       " '▁that',\n",
+       " '▁describes',\n",
+       " '▁a',\n",
+       " '▁task',\n",
+       " '.',\n",
+       " '▁Write',\n",
+       " '▁a',\n",
+       " '▁response',\n",
+       " '▁that',\n",
+       " '▁appropri',\n",
+       " 'ately',\n",
+       " '▁comple',\n",
+       " 'tes',\n",
+       " '▁the',\n",
+       " '▁request',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Inst',\n",
+       " 'ruction',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'Det',\n",
+       " 'erm',\n",
+       " 'ine',\n",
+       " '▁core',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '▁detection',\n",
+       " '▁of',\n",
+       " '▁following',\n",
+       " '▁d',\n",
+       " 'na',\n",
+       " '▁sequence',\n",
+       " ',',\n",
+       " '▁The',\n",
+       " '▁result',\n",
+       " '▁will',\n",
+       " '▁be',\n",
+       " '▁one',\n",
+       " '▁of',\n",
+       " '▁the',\n",
+       " '▁following',\n",
+       " ':',\n",
+       " '▁Non',\n",
+       " '-',\n",
+       " 'prom',\n",
+       " 'oter',\n",
+       " ',',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Input',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'CCG',\n",
+       " 'TGCG',\n",
+       " 'ACCGG',\n",
+       " 'AAG',\n",
+       " 'TGGGGC',\n",
+       " 'GGCG',\n",
+       " 'ACCCCGG',\n",
+       " 'AAG',\n",
+       " 'TCCCC',\n",
+       " 'GCCGGG',\n",
+       " 'TGCAGC',\n",
+       " 'TTGG',\n",
+       " 'TCGG',\n",
+       " 'TTCG',\n",
+       " 'ATCGCC',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Response',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'prom',\n",
+       " 'oter']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenizer.tokenize(prompt)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "11875339-4901-4912-86e5-afe8c74921d9",
    "metadata": {},
    "outputs": [],
@@ -498,10 +718,48 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "1b02644a-8b24-45aa-b22d-0f7ce2270dd9",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "real answer: promoter\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/configuration_utils.py:601: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.01` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "input_text = format_input(data[\"test\"][0])\n",
     "\n",
@@ -562,10 +820,118 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "id": "68831e19-5a99-46d8-9f40-e8bf6957dbfc",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Donor Sites |||||||||||| Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "presicion 0.73 same 0.3\n"
+     ]
+    }
+   ],
    "source": [
     "import json\n",
     "from tqdm import tqdm\n",
@@ -598,6 +964,14 @@
     "\n",
     "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bc38f47-4a7d-43eb-abe8-db4310d280e3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/gpt2-small3-1024-checkpoint.json

@@ -0,0 +1,602 @@
+[
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGTCATGCCTGCAGGGAGAGAGGGCTGGGGCGCTATCTATGGTAGAAGTCAGAAGAGGCTGTCTATATGTCAGTGGGAGGAGAAAAGTGGCTCCAAAGCGGACTGTCTGTCCCTGTGCCAGAAGACATTGCTATTCAACGTTCACTCGCTTCTGCACAGGCCCACGTCACCCACAGACCGAGAACTCATCCACTAAGCGCTGCGGGTGGGCAGTACATACCGACGAAGTGCCTTCGCTACCCCTCTGGGTGTCCGTGCTCCCGCGAACCGGCTGGGGCTGCAGACGGGAAAGGAATGTCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGGGTTTGCACTAACCGATCCACTCCGCGGGGACCTCTTAGAATTCTGCTGTGGATAAGGATATCGAACTACCGAGTCACTGGACCTAAAGAGAGATTCACTGTATCATCGTCTGCCTTAGACTGTTCCGCACCCAGGAACTGGGGCTGTTGCGCCCTTAAGGTTACTTTGAAGCCAAGGTCGCAAACAGACTTCCGCATTGACGTCAGTAGCCGAACGCTGATTTTCTTAATCTAGTATTTAGGATGGGCCTCTGTCGCCTAGCCGCTATCGCAGAGTGGAGCGGGGCTGGGAGCAAAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCTCGTAAGGACCTAACTGTTCCAGCACTACACAGCTCATGGTGTCCTCAAGATTAGTACATATCAGAGGCCATGAACTTAGTCTTACGCTCACTGTGGCACTGGACGCGCGGGGGATGCGGGGTTGTTCCCAGCGATTTATGATTCGTTATTTGCCGCGTTGACTCGCCGTCTGTAGCCCCATGACAACGACATTCCTGCATTCTCTGCCTGGAGAGCGAAGTGACAATACTGAATTGAAACGGCTAGAATGTCGCTCGCTGAGGCTCCGGACCTTGGAGCGTCTAGAGTCTGGCTAC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGCCACATAAGTCGGGTCAGCAAGTCGGTGGCGAAAAGAGGGCCACGTCCCAGACTTTCTGGGAGGAGGCGGGACAAAGGGGCGGGGCGAGCGCAGCGCCCTCCGGGTGGGCGCCGTCAAGAGGCCGGGGGCGGGGCCGAGCGCGGCTGGGCGGGGCCTTGAGAGGCCGGCCGGGGGCGGGGAGGCTGGCGGGTCGGCGCGGGCCCAGCCGTGCGTGCTCACGTGACGGGTCCGCGAGGCCCAGCTCGCGCAGTCGTTCGGGTGAGCGAAGATGGCGGCCGAGAGGGAACCTCCTCCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATAGGATGCGTCTGTACTCTGATTAACAAACTGCTGAAGGAATAAATATGTACTTGCTGGGCAGCGCCGCCGGCCGAATGGAGATAAGCCTATGCAGCTTCGTGCGCGGCTCCCCAGCCCTTTGCTGCGCCGCGAGCTGCGCCCTGAGACCCCCGCCTCGCTGCCAGCTACTTACCTGCCCCGGCGGAGGGGGCCATGTTGCTACACCTAGGCAGGCGGCAAGAAAGCACGCGTAATGAATTCCTTATATCCCCCGCGCCCCAACGGCGGCGGCGCGCCGGCCGGCATGGAGCCCCGCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGGGTAAAAAAGGGGACAAATGGAAATTTGTGTTGCAGATAGATGAGGAGCCAACAAAAAAGAGCCTCAGGATCCAGCACACATTATCACAAACTTAGTGTCCATCCATCACTGCTGACCCTCTCCGGACCTGACTCCACCCCTGAGGGACACAGGTCAGCCTTGACCAATGACTTTTAAGTACCATGGAGAACAGGGGGCCAGAACTTCGGCAGTAAAGAATAAAAGGCCAGACAGAGAGGCAGCAGCACATATCTGCTTCCGACACAGCTGCAATCACTAGCAAGCTCTCAGGCCTGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCTGGGTCCTAGAGGCTACTGAGACAAATGCTGGCTGAAGCCGAGGTACCTAGATTTGTGCAAATCGAGGAGTTCTATTCTGCGTGTTTGTGAAAGGGGGTTAGACGAGAGGAGTGCGGTGGAGGCGAAGATAATAGGATTATTCTTACAGTGTAATAATAATTATTGGTGCTAATGTTTAGGAAGCCGTGAAAGATAACACGAGGGGAGAAAGTGGATAGAAAAACCACCGCTGGCAGTTCTTTTCATTTGAAAGTGAATTGGGAAAAAGCGAATAAAAATTAGTCATGAGTTTTGGTA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTCGTTGCGTAATCCTCCTTCCGCACCAACGCAAGGCAGGACAGGAGGTCAAATGAGTTCGAGATCAGGCGCCACAGACTCCACTAAAGCACAGACACCAGGTGCGGTAAGACACGCTGGCTCCAAAGTAGGACACCAAACCATTGCTAATGCCTGGGGAGGGCCAAACATGGGCAGGCTGGAGCACTCTCCTCAAAGAGGACCACAAGCCGTGACAGCAGAGCCCCTCGGACTGATGGGACACGGCTTAAACTGAAGTTCGAGGACTCTGTTCTGCAGAGGTCTGAGGCTGGAAGGGGG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGGAGAGAGCTGGAGTCTCGCGGCCTCGGAGGCCGGGAAGCCAGCCCCGGCGGGGACCCGGGCGGGGCGGAGCCAGCGGAGGCCCCACCCCCGGCGTCACCGGCCCCCAGGGGGGCGTCGCCCCCACCCCGCGCTCCAGGTAGCGGCGGCCTCAGCTGCCGGCGGTGGCTGGCTGAGCTCCCCAGCGGCCCGCGGGCCGGGGCGGGGGCGGGCCGGGGGCGTTCCCGCGGGCCACCGCCCGTGATGTCACAATCGCGGCGGGCCGCGGCGCTCCGGGGTCGGCGGTGGGCGGCGGGCCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGACTATACCACTTGCCGGCCGCGCAGAGCCCTCCCCAGGTCGCGCAAGGAGCTCGAGCTACTAACGGAGGAGCGGCGCAGGCCCCGCCCTTAGTTCATGAGCTTACTTGTACAGATCTCGGCCCCGGCCCCGGCCCCGCCCCCTGCCTCATCGTGGACCATCACGCTTCCAGCCCAGTCGGCTACGGAGATAATCCATGGAGCTCCGAGGCGCCGAGAGAGCAACCTTAAAGTATCGGGATCAGCGACATTATCAGCGCTGGTCCCTGCAGCTTCTGGTTCGAGATCTGGAAGGGCGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGTTTAAAAATGAACCCCCAACAGAGCCACCCGCCGGGGACGGTTCTGTTTTCAGCACAACCACGGGGTTCGTTTAGCCTGTCGCCCAAGTTTGTTTCCTCTAGGTCGCCCACCTTCCGTCGAATTCTGTCACTAGACTTTTATACTGGGTTTGTATGTAGCTCGATTCTGCATACAAGCTGGACTGAGAACTGGCAGGAGGCGAAGATGAGGGGGCGGCATCCAATTTGTTGTGATCCTTTTGCGTCAGGCTTCTGCCTGAGCTCGGTGAGGTCAAGCCTCCTCTGCTTCCACCCCTC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCCGCTCCGCGGGGCAGGCGTCGTGGGCCCAAGGAGGCCAGGCCAGGAGTCTCGGCCTCGGGGTTACGCGGCGGCGGGCAGCTCTTCCGCTCCGCTCAGGGTGCTGCTAACCCACAGCCAAGCAGCGACCCGCAGGACCGGCGAGAACCCCGCCACTGACTCCGGCCGGAAGAGAACGTGCAGAGATGTACGTCACTTCCGGGGCGTGGCGTGGCCGGGCGTGGCCGTCCCGCCCCTGGCGGCGTGCCTGGTGCCCTGTTCCGCGTCTGTGCGACCGTCCGTCCCGAGCGCGCAGCCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TACCGGTGCTACTTTACGCTGCGTGTTCCCTAGCAGGGGACCTAGTGTGCTACCATAAAAGCACGACGATAAGACTATACCTCTTCGCCTGCTACTAACATAGGGCACTTCTTCCCCCTCAATCTATCGACGTTCGTTAATATCACCGACTCCCAGGGAGGATCTAATAAAAGGAGGAGGGCTCCTGGAATAATAGAGGAGTGGTGCTGCGAGGGGAAGGAAAAGCCAATCCGTGACCCCTTTCCCGGAAAGCTTCGAACCGACAGCCAGAAGCCCGCACGCACCTCGCACCATGAGATG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCGAGAAGACCTGGGCTACTGAATGACGAATGGGACGCACACGTACTGGCTACTTTTGTGGAAGATGGTCCGCCCTGGTTTGGATTGCGCTTTGCCACGAGAACCGCCGGGAAGGTGGGGGACGGGGACGGGAGCAGCCAATCACGCGCCGTCCCGTTCGCTTCGTCAGGGCGCGATCGCGGATTCGGCACTCAGTGTATGGTACTTCGGGCGAGCGAGGAGTCGCGAGCGCTGTGAGTTTCTCCTGGAGTTTTCTGCCGTTTTCGGACCCTAAAGCTTATCTAGGCTCGCGCTCTGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCGTCCTGCTCCAATTGCCAGGAGATACCATTAGCGGACTGACTCCAAGAAAAGAGATCCCTTGAAAAAGGCAGTGCTGAACGCGAGTATGGCGCTGCTTGCGGGCCATTCTGTGCGCTACGCTACCCGCTCGTATTAATCAACAACAGGACCGGACTCTGGTTGCTTGGGAGTTCAGGTGGACAACAGCAACTACTAACAAGGTTTTTCGTTCATAGTGCGGGAATGGCAAAACACGAGAGAAACGCACTCTTCATAGGTCACTATCCGTCTGGAAGAAGGCTTTGGACAGAACAGGG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTGCCCCCAGGCACTGACAGGGTAGCAGAGGGAAGACACATGAGCTGCTTTTTAATTTTTTTTAGAATTAATAGAGACGGGGTCTCACTCTGTCACTATGTTGCCCAGGCTAGTTTTGAACTCCTGGCCGCAAGCAGTCCTCCTACCTCAGCCTCCCAAAATGCTGGGATTACAGGTGTGAGCCACCGTGCTTAGCCATGAATTGATTTTAAAACCTTAAATGAGGGGCCAGGTGTTGTCTTTCCGTTTGCCTGAGTGACGCGGGTCTTCTCCAGGACATCCGGCCACAGATCTGCC",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CAGCCCCTATGTTGGTGTGGAGACCCTCACCCGAGCTGGGCCTGCACCCTACACCAATCCGAGGTACCACGATCGGATTACTACAACAGCGGGGTCAGCTCAGGCGCTCGGTCCCCCGGGATCCTGCTCCCAGGCACACGATTCCGTTCGTTATAATTATGTAGACGGGGCCTGCGGTAGACCGACTGCGGGTGACCGAGTAGTGCAAAGTCTGGCTTCTCTTTTGATAGAGCACATCGATCTGTCTTCCTCTTTTCACCGTCGATGAGCCCCGCGTGCCTGGGACTGCTCCAGGCTCCA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCGCCGCCCACATTAGTGTGGGGCCCTGCGGCCTAGCGTCCCTCACCAGAGGCCTCCCCTTGCCTAGCTGGACCGCCGAGGGACATCGACGAGTATCCTCCTCCTGCTGTCCCCGGCTTCGCCTGCCGCCCCTAACCGGCCAGTCAAGATGGCCGCCGCTGGGTGAGGCAAGCTGGCGCGCCGCGGGGGCGTCTGGGAGTTGTAGTTCGGGACGGCGGGCTGACGCACTTCGCCGCCGGCCGACGGGCGCCATTGTGCGGCGCGCGCCGGGTGAGTGCCGCGCGAAACCTGCGTCCGTC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCGGGTAGCGATGGGCCGGGCCTCAGTTGTTTAAAGGCCCGATGCGACCCCGCTTAAGACGTCAGATCGTTCACCCCCGAGGGTCGCCGGATTTCGCTCCCGCGCGAGATGGCCTCCACCAGAGACTGCTGCTGCGCCTTGGAGCGGTGCCTCGCCTCCCAATCCTCGTCGTGAATTGCCCCAATTGCTGATCTGGGAACAAGCTCGTGCCAGGCGGGCGGAGAGTACGAACTCCTTATAGACCAGGATGGGCGTAGCAAGAGCGCGGGCGGCCCATGAATGTCTGCAGCGGTGAGGACC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGATGCGGGGCCGGAGGGGATGGAAGGGGCCGATCTAACACGATATTCCGTTCAAGAACTACAACCGTCTACAGCACACCATAGCCGCTCGCTGAAGGATGTCGAAGGGCGCCCAGGCACCCACCACGTTTGGCCCCTGCCCGGGCCGGTTCCGATATGACTCACTCATTTACTCCCCTAGAGCTCCGAGCCTCGGATCGCCGAGGTGACCCTTACCCGCCTTCCTACCACCTTAGGGGATTTGGCCCGGAGAACGAGATCACCCTCTCATCGTGGCGGGGTATTCCCTTTAAGGTTTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCCGTCGGTCCGCGGGGCTCTCGCTGGAGCCTCCCCGACTCCGGTTTCCCCTCGTCCAGAAGCCCGACGTAACCAAAGCCCAGTCTGTCACTTTAAACACGCCCCGCCCCGCCTCCCGCGGCTGTGTTGCCTCCTCGCTGGAGAACACCCTGGTCGACCTCTGTGCGTCCGTGTGCGCGAGCGCGTCCCGCCGAGGCGGTGGGCAGGGCGGACGGTGCGCAGTGCGTTCCCGCTGGTCGGAGCCAGCACACTAACCACGCCACGCGCCCTGCCGTCCCTTCGCCTCCAGCCGCTGCAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACCTACTCGGGGTCCGAGGAGGGAGGATTGCCTAGGCCCAGGACTTTGAGAACACGCCTCTACAGAGATTTACATTTTAAGAAAATTAGCTGAGTTCGGGTCGGGCGCAGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCTGGCGGATCACCTGAGGTCAGGAATTCGAGACCAGCCTGACCAAAACGGCGAAACCCGTTCTCTACTAAAAATACAAACACTAGCCGGGCGTGGTGGCAGTCGCCTGTAATCCCAGCTACTCCAGAAGCTGAGGCAGGAGAATCGCTT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCGGCCGGTGTGCGGGACACACTATTGTAATCCTAAAAGGCCATGGGCCCCTGCTGGGGTCAAGAATAATGCTCAAAGTTTAAACTACCGAGTGTTGCTGAGGCTTGGGTTGCACATCGAGAGTCGCGATAACACTCACCCAGGGGAACATACCGCGAGGGATTACGGACGATGGAGAGATCACCCAAGGCAGTGGGAGTCCGCGCTTCCTCCCTTAAGTACCGCTGGTTGTCCCCCGACGTAGTGTAGATAGTGAGCCCTCTTGTCCGCTGGCACGTGATGTATGTGTGTCCTTGTCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGGTTAGGCGCGTGCCGCGAGAACAGAGTGGACGGAGCGTAGGAGAGACCGAAAAGGCTGGGGGTGGGAGTAGCGGATTTGAAGCACTTGTTGGCCTACAGAGGTGTGGCAAGCAGAGCACCTCAGAACTCAGGCGTACTGCCCGCCGCCCGAGCCCTGCGAGGGCCGATAGCGAGGGTGTGGCCCTTATCTGCACCCAGCAGAGCGCCGGCGGGGTACGGTCTTAGGACCTCGATCTCCTTCTCCCTCATTTTCTCTCATCCCTACCTATTGTGGGTGAGTCCTGGCCCCTGGACGGGG",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTTGCATTAATACACCCGGCGCCAGAGGAACGTGGGGATCAATCATATTACACTGCTTCAGACTTGTAATTTTATTTGGTTCAGGCATCGTAGTGGTGAGGCTTGGCAGAGAGCCCGAGATGGAGCTATTGAGACGTTTCAGGCTGGGCGGATAATCGCATCAGTAGGGACTTGGCTAGATCTCCCAAGCCTTCTCCACTCTTCAAACTTTCACCAAACTCTTAGGACCCGTCGGGTCGGTTTCATAGACTCAGCCGCTCAACCTTTTAAGCGAGCGGCGCGGCGGGGCGGGCGAACTCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGAGCGTCATCTTATCATTCTCTCTTTTTTTTTTTTTTCTGGTAATGATTTTTTTTGTTTTTTTTTTTTTTGATATTGACAAAAGTTTAATCATTTCAATTAAAAATGCCACTAATTTGACTTTTTAAGTAAAAAATGTAGGGGGTTTTAAAACTACTTTCCTACTACCAAAAAATCAGAAAGTATCTAGCTTTCTAAATTGGGAAAGCAAGCAATGTTATAAAAACACTGAAGGAATCTCTTTCTTCGTGACCTTTTGTTAAACTCGGTTTAAGCTGTAGACCTTATTTAAAATAAAAT",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGCTGTGAAACAGGGCCACCTGCAACCGTGGCTTTTCAAAGCTGAAGCCATACCCTGGAGGGAGAGAGGATCCTTGGGCACTGACCCTGGCTGTCCACAGCTAAATGATCATGGGACACCCATGACCATGGGCGTGGCTGGAGGAGGCCCTGCCCTCAGGGGTCTATTGGCATCCAGGGGAAGTGTCTACGGGGTAATGATTAACCCACACCTGCCACCTGGTTTTCAGCCTTCTTGCCCTGCTGACTCATTTTGCAAATCCCACTTGCAAAATTTGGGAGCAGCCTGGGTCCAGCCCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTGTGACAATCATGGAGCAGTTGATTGACCCCTTTTCGAGCGCTCGCTATGCTTTAATTTCAAGACCCGCTCTCTAAGGTGCTCTTCCCCTTTAAGACTCACGGCTCTTCCCGTCAGGCTAGCTCGACCTGAGGATTCGTCAACCTCCTTGACAGCTTCCGCTGGCCTGCACCCGAGATTGGGGACGGTTTTCCGCACATCGTGCCTTCATTAGTTAGGTCCTGCGAGTGCATCCAAGGGCCCTCCCCCAAGCCAACAGCAGAACCAGCTCAATTTCTACCCTCACGGCGCTCCGCGGTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TAGAGAGGCCAGAATCCCTCGGGAGTGCTACCCGGTCAAAAAATCGCGCCGCCCCGTGGGGGACCGAACTTAGCGATGACGGTTCGTCTAATCCCGTAGGAGAGTATCGGGCGAGCTCGGATTGCGAGATTACCTAGTGTTAGTGCGACCGCCCAGGCACATCTGCAGACTGACGTCAGTATGCAAATCAGAGTCGGAAAAGCTGCCAAAATTTTCTCCTGCTGCGCCACCTAACCTGCGTTCGGGGGTAAACCCATCTAGGGCTACTTCAACAAAACTTTGCTGCCCTTCCTGCTCCTC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGCTTTGTTTAGGTCCGGAAGGCGGGCTTTCCTGGGAGTGGGTGGGGAGGGGGCGTTGATTCTTGACCAATCCTTTCAGTCCGTTGGGTGGTGACCAGCCAATGGGCCGGATGGATAGGACGCTCCTCCCGGAGAGTAGTGAGACCCCTGGTGCGGGGCGATTGGCGGCGGGAGCGATGAGTGGCAGCCGCACGGCCCAACGGGAGCTGTGCGTGGGCCGCGGGGCGGGGCCAGGGCGGGTGCGCGGCGGCGGCGGGGTGGCTGGGCCGGCGGCGGCGGCGGTACGAGGCGCGCGCTCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCATCCAGCGAGAGGGGCAGGTTCCGCATTTTCTCTTCCCCTTTCCCAGCGCTTCCTCCAGCACCCGAAGCCCCAACCCTGCGGGTCAGGAACTCCCTAGTCCCCAAGTCTAGGGATGAGATGGGGGAAGGAGAGCCGTCAGGGTTGACCTGGAGTTTTGTCCGCTCCTCCCCTACAGTGATCCCTCTAGCCTTCTCCAGTCGCCTCCGCCATGTCCGAGGAGCTGGCCCAGGGCCCCAAGGAGAGCCCCCCGGCGCCGCGTGCGGGCCCCAGGGAGGTGTGGAAGAAGGGTGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACATCGTATTGCGCTTACTCGGGTCGTGCAATGCACTGACTAGGAAAGGGCTGCGGATGCCTACGTACATAATGCTGTTGCGCACCTATGCACGCGGGTCTGGTCCTGGGCAAGAACCGCCCCCTCTCCGGGCCTGGTTGCCTTTTGATTTGCAGAACAACGGGCCAGGCCCCTTCCCTCTACCGCTCACAAGCGTACATCGCGACCATACCTGGTCCAATGCGCTACGGACGAGGGCAATTCTCGTAATAGGACCCACAGCAAGTCTTGTATCACCTTTGGCTAGCCTGTGCCGGCTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACGTGACCGACGTCTTCCGCATACCCGGGGCTTCCCTCACTTCCAGGACAACCTGTTGCGCCTCTATCTCCTCCCCTCCCCGCAGTTTCCCCGCCTTGGCCTCTAATAGTAGTGAGAGCACTTTAGGCGGTATAAAGTCTGACGCTAGAATCAATCTAGCCGCGGCGATTGGCCACCTGGGGAATCGTTAGCAAAATGTCACGTTTACTACTTTACGGCGATCTTCCAGGGTCCGGGGACTGTGCCAAGGATCCTACCGGGGCTGGCAGGAGCCTAAGACAAATACGCTAGCGGCGGAGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAGATTTCACGCCTAGCGGCTCAGGATCGTGTGTGCTACAAAAATAGGGCCGCAGGGCCGTATAGTCGCACGAACATACAGGTCGTCCGATTCAGGCTCGGGGGCTTGGACGGAGATGACGGCCAACTCTCACCCTCACCCCTCATTTAGGCGGGCGCCGGGGGCATCATCGGGGTCCCTGGGCCCTCCCCCTGCTCAGTGCGTAGCTCCGATAGAAAAGCTAGGGCATACGTGCAGAGACGGGTTGTCAGGCAGACACCCTGTCTGTTGGCAGCTAGCAGCTGGCGGACGCGACCCGGA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAACCATGCAATCCGGATGTGGCAAACAAAGGGTGGGAGCCCACTTGTCATGGCATTAATTGCACGTCTGCTTAGGGCGAGTTTTAACCGTGGCACACAAAACAGCACGATCGAGGATACGGGAGGTCGCCGCGTGATATAACGTCTACCCATGGTTCTCATCGAACCTCTACCTAGACACTGACCAAACCAACGCTCCCAGTGCGCACGGCCATTTCTCATAGATATCTTGAGCAGGTTGCACCACCAAGATAGTGAGCAGGGCGGTGGGTGGTGCGGAGTGCCGAGCGGCCTCACCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCGGAGGTCGGCGGCCAGAAGCCAGATTCCCTTGGCCCCAGCTCGTCCACTTCCATCCAGCAGTAGCCAAACAACCGTTGAAAATGGCCTGGCGGCGGCCGGGAAACACCAGAGAACTATCTTCCAACCCCTAGATCCCGCCCATCTCGTTCCGGCGGTGTTTCCGTGGCGACGCTATCCGAAGTGCGGCTGCGCAAGGGTGACGGCGCGCGAGCAAGGGGGAGGGGGTGTTTTGGTTCTAGCCGCTCGCCGTCCTTGCAGGCTCTGCCGTCGGAAAGCCGCTCATTCTCGCTTCCCCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTGGTTGTAGGTGGGAAGAATCCTGGGTTCCCATGCTGTCTGAACCAGAGTCATTTGTGGCCCTGCCCTCCCCTGGGACTCAATTTCCCCACCTATAAAATAAGCCCCAGTGTCCGCGAACCCTGGAGGGGCCCGCACCACTGCAGGAGCGGCCGCCGGCGCCAGGGGGCGCCTCCTCGATAACTCGGCGCTCGGCTGGCCAGGCACCGGCGCGTCGGCCGCTCGATTGGTCGGGTCGGGGCCGGCCTGAGCGCCGCGGGCCTGCGCCATTGAGGAGCGGCGGGGAGGAAACGCCGCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TAGGAGGCCTGTCTGCACAGCTGTGCTGGGAAGCTGATCCATTCCAACCACCGCCTCATGGCCGCTTACGTGCAGCTCATGCCTGCCCTGGTACAGCGCCGCATCGCAGACTACGAGGCTGCCTCGGCTGTGCCAGGCGTTGCTGCTGAACAGCCTGGGGTCTCTCCATCAGGCAGCTAGCCATACCCAACCCCAGGAAGGAAGGCCTTGGATGGACCCTCAGATTGAAGGACCCGGTGGACCTTGGGGTTGGTGAATCCTAAACAGAGAGAATTCGAGGTTGCCTGAAAGCTGGGTGTC",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTTCTAGCTATCTTGAAATACGCAATACTTTGACATTAACACACTGCTATGTTCTCTAATGACATCCCTGGAAACCTCCTAGGGGCAGCCAGATCTTTCATGATAGTGGTTGTCAGTCCTCATATGGAGGGTGGAGGTTTGAAGCAGAGAGCCAAGGGAGGTTTTGTGCACCTATGCTTGTTGTGTTTGTACACAATGACTATGCATACGCTGTGAGTATAAAAGGCTCATTTAATCCTATTGTGTCCCAGGCTTGGTTTGTTTTCAAATCATTACAGCATGAATTTAGAAGGTTTTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCGAAACCCCTCATCTTGGGGGGCCCACGAGACCTCTGAGACAGGAACTGCGAAATGCTCACGAGATTAGGACACGCGCCAAGGCGGGGGCAGGGAGCTGCGAGCGCTGGGGACGCAGCCGGGCGGCCGCAGAAGCGCCCAGGCCCGCGCGCCACCCCTCTGGCGCCACCGTGGTTGAGCCCGTGACGTTTACACTCATTCATAAAACGCTTGTTATAAAAGCAGTGGCTGCGGCGCCTCGTACTCCAACCGCATCTGCAGCGAGCATCTGAGAAGCCAAGACTGAGCCGGCGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCATCTTAGAGTTTCATACAAGGATCAGATTTGTGCTATCGGAGGCCGCCGATCACCCAAGTAAGCTTCATACGGGGAGGGAGGGAGGGCGCGAGCTTGGGAGTGGAGAGGGAAGGAGCAAACTTTGGCATGTATCACAGGGTAGCTCTGCCCCTGCGCGCCCCCTCCAGATCCGGCCCGCGGCTCCCCTCCCGCGGGACATGCTGCGGGGAAGCACGGCGAACAGCCCCTCCTATGATCACAAATGGCTCACCGCTCTCGGAGCCCGGCCGCCACCGCCTGGTGTGCGAGGCTATCCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCTCTGACTCGGTTTCCCCTCTCCCCCGGCTCCGCGGTCGCCCTCCTCCTGGACTGCTTATTTCGTCCTCGGCCACTGCCTCTCGGGCTCGAGCTTTCTGCTTCAGAGCAGGAGAGAAACGGAGCGAAAGCAGTTTCTGTCTCCCCGGGGTCTGACTCGGTCCCCCGCGCTCGGTTCTCTGTCCCCTCCCCCTCCCGCCATTGTTCCCGGCAGGAGGAGTGCGCGGCGCAAACTTGCGAGTTCCCCCAGTCTTTGCCCCCGCAGGGCCGCGGACGGAGGTGGTGGCGGAGTTCCCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGCATGATGTTGGCCGACTTGTGCTAAGGTCCTTGGAGAGGTGGAGGCTCTACGGGGGGTGGGCAGTACCCCTTAGAAAACACCCCTCACTCTCAACAACAGAGGCGGTCTGACGTATTGGAACTCATTCTTTAAGGCCAGAGCTGTGGTTGCTCCAGCTGCAATGTGCGCTCTGATGATACTTTATAATAGATTTGAAAAGGGGAGGGACAAGCCTAAGGTCATGGCAGGAGGCTGGGCCAGGGAAGTGTGGGATCCCGTGGTTGTAGTCGGCGAACCGAACGTCTCGGCCTCTCTCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGGGGGTTTTTTGGGGGCTCGGGCGTCGACTGTGGGGCGCTGCGGAAGGGTGGGTCGTCCGCGTGGGGTCCTGGAGCGCAGGAGGCGGAGGGGAAGCCGCGCAGGGGCCACTAGGGAGGGGCTCTGGCGGGGTCGCAGGGGCGCGGTCTGCGGGCGTGGGGGCGGGACCTCGGGGGCGGGGCCTGAGGGCGCGCGCACGCCTCCGGCGCGCCCCCTCCCGGCCGCCATGTTGGCTGGTGTGTGGGTGTCAAACTGAGCCAGACGCGGCGGTGGCGGCGGCTCCGCGGGCTACGGTCGCTC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGAGAGGGGCGGTACGCACCACGGGGGAAGCCAATGAGAAAATCAGGCCCAGCCCTAGGGGGCGGTGCTGTCGGTCACATGCGCACCTGGGGCGGGTGGTGGCGGCGGCGCGGGCACCGCGAGCCGGCGGAAGGGAGAGGGGCCGGCCTGGGGCGGGGTTAGGCAGGTGAGTGACAGGCTCCAGGGGGCCGGCCCCTGCCTGGTGCCCCGAGCGAGCCGGGAGTAGCTGCGGCGGTGCCCGCCCCCTCTCTCCGCCCCTCCAGCGGAGCTGGTCTCCGGCCGGGCACCGTCGCGGGCCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGACAACCCGCCCTTGTTCCCGGTAGAGGAACAAAGAGTGCTGATTTACGAATCTAGGCTGCCAGATTCTGTGTTTCCACTGGAGGGCGCGTCCGATGCAGATGTTGGCTCAAATTCCATCTTAACCTATAAACTCAGTTCTAGCGAATACTTCGGGCTAGATGTGAAAATAAACAGTGATGACAATAAACAAATTGGGCTCTTATTAAAGAAATCCTTGGACAGAGAGGAAGCTCCTGCACACAACTTATTCCTGACAGCCACAGATGGGGGCAAACCTGAGCTCACAGGCACTGTTCA",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TTCCAGCACTCCGGGAGCCCGATTGCGAGTTTGTGAGAGTCCTTGGTGGAGGAGGTTGGAAAGCGTTCTCCCTCCCACAAAACCCGGTGCAGAGGGAGGGAGAGCCAGGTCCGTAGCGTGGGTCCAAGAAACTATGGATCCTGGAAGTGAGCACACTTTTGTGTCTTAAAAAGTTTGGTGTCTCGTTTTGAGGTGAAAAATGAATGAGATAAGTGCTTTTTAAAAAGGTAAAGTTTGTCTTACAACTGAAAAAGCTTTTAAACAAAACTATAGTTTTTGCGAGTAACTCCCGTGACACCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATCAGCCCTGGCTAGGACCAGCTTCCCACTAAATCGCTTGCGGCAAGGAAGTCTCAGCCCCCGAGGCAGTCCTCACTGTGAGCAAACCAAAAGATGGGATCACTTGGGGCGCGTACGGGCTCCCAGCGCTGGCAGAAACTGTAGCGTTGTAGTAGCCTTTGTGACTGGTTGCCTGTTGGAAGTCTACCCCACCCCCAAACCATAAATCCACCGGAAACTCCTCCAGGACACCCTCTCAAGCAAGCCCCTGGGATGTAACTGCCACAGGGGCTGCGCCACGATGTTCCCTGGCTCTCAGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACCGCGTGAGGATGTGCCGGGTGGTCCTTTCCTCCTCCTCTTCCTCCTCCTCCCGGCTCCCTGCCTAGTCTCCATATAAAAGCGGCGCCGCCTCCCCGCCCTCTCTCACTCCCCGCTCCTCTCCGCCGCGCACTCTCCGCGGCGCTGGGAGAGGGCGGAGGGGGAGGCGGCGCGCGGCGCCAGAGGAGGGGGGACGCAGGGGGCGGAGCGGAGACAGTACCTTCGGAGATAATCCTTTCTCCTGCCGCAGTGGAGAGGAGCGGCCGGAGCGAGACACTTCGCCGAGGCACAGCAGCCGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCAGGAGTCGGTAGGGGCAAGAACGTTTCCGGCTCCTCAACGGGAGAAGCGCGGGAAATTTGTTGACGTGGAATAGCTCGACGAAATCATCGGCAGGCGCCCGCCAGGGAGCTAGCAACGGAGCATATGCGACCGTCGGTTCGCAAGAGACACACCTCGGGCCTTGCCCCGGAACGCCGCAATGGCTCCTACCTTTCCGCCGAGTGCAGGTTGATGCTGCACACCCAGAACACTTTGCGGCACGGGGGGGGTTATATGACATGCAAAGCCCTGATTCAACTTGGAGATGAGCGGGTCTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAACTCCTGAACTCAGGTGATCCGCTTGCCTCGGCCTCCCAAAATGCTGGGATTACAGGCGTGAGCCACTGCGCCGGCATCCTGGATGCAACACGATCAACGAATGATTGCCCTTTATTCTGCCTGGTCTCCCAGCGGTGCTGTTGTGTCTTCATTTTAATGGATTTTAATTGATGTCACTTAGGGGGGCCCTCAAGCCTTCCTTCCTCTATCCATTTACGTGGTGTACCATCGAACTTTATCGACTCAGAAGTCGTGGGCGGAGGGCGTTCCGATAGAACCAATTGGCTCACGTCTATA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGATGGCCTGAGTCAGGAGCAGGGGCAGAGCTCAGGTTGTCGTGGGGTCCCGCTGCCCACGTCAGACTGGAGGTGAGGGATGGGCGGGGCCTGACAGCAGGCCTGGAAGGAACAGGATGTCTATGCTGGAGATAGAGGGAGAGGACAGTGCCAAAACCCAGCTCCTGGCCAGTCCCCAGCTCCTCCCTGCCTGGCCCTATCCCAGGATCCCCTCCCCGGCCTCCCAGCTATGATCTACCCCGGGGCCCAGACTTCAGGCGCCTTCACGATGCCGGCGGTCAGTGGTCCAGGTCCCTTAT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCTGACACCCTTCGGCCATTTTTTTTAAACGTGTTGGGGTTTGGGTTTCAGCCACAGGCCTAGCACATGGAACCAAAACGCTACCACGGACTATGGAGTTGTGAACCGCGGAACAGCCGGGCGGACTCCGGGGCTCCTGTGTTTGAGATACGCGTGCCATGAACCTCCGCGCCGGAGGCGAGCCACTGGTAATTTACTTTCCAGTTGCGCCAATCCGAGTCTACCTTGATATTGAAAGGTAAGTCAAGAACAAAATTATCGCATAAGACTGTTGAGGTTACCCCGCGGCGGGGTCGGGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTGAGGCTGGCAGGCACTGGGCAGGCAATGAGTTCTTTCCCTGTAAGTTGGGCACAAAAAGATCGGCATGGGAGCAGCCTCCTTTGAGACAGCTGCTCTGAGAGAATGCAATAAGCAGGGAGCAGCCAGCAATTCCTCCTAGCAGAGGGCGACTCGTGGGAGGAGTTCAGTTTGCCAAGTATTGTCATTTGTTGAGAGAAGGTGTGTGCTCAAGGAGGAGTTTTAACCTGGAGGATCATTAACTCTTTTAGTCAGCTGAGGAGCTGCGGTGGCTCGGCGAGTTGGAGTTCATCCTGGAAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCGCGCCCCTCCCCCCGGGTCGCGCGCCGCGGGCCCGAGCGCGAGACGCCGCTGCTCCCGCCCCCGCCGCCTCCTGGAGTCGCCGGGGCGGACGCGCAGTCCATGGGGCGCGGTGGGCCGGGGAGTTGCCCCAGGGGCCGCGGGAGTTGCTGAGAGGAGACAGGGTTGGGCTTTCTCCTCGCCCAGACCCCACCCCACCCTGCCGACCCCACCCCCTGCTCCTTCCTCCCCGGGGGCGCGCACTCGGGCACGCGCTCGGAAGTCGGGGGTCGGCGCGGAGTGCAGGCTGCTCCCGGGGTA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATCTCCCCTCCTCGGTCCCTCTCCCTCCCTTCCCCCTCCCTTCCCCCGCCCTCCCCCTCTTTTTGCTCCTGCTCCCCCCCCCCACCCCGCCTTTCTCCTTTTGCAAGAAAATAATTTGACAGTCGATTTGCTGACAAGGGAGGAATTTGCATCCTGGATTTAAAAAAAAAAAAAAGGCCGAGAGGAGCTTGGGAACGGTTGCTAGGGGTGGGTAATGGGTGAAAAAAGGGGGGTACCGGGGAGCGGATAAGGAGGGTTAAGGGAGGGGGCGAGGATGGGGAGCAATGCAAAGGTAAGGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCAGTAAGTCGGGGGGGAGGGAGCCGGGGCCTGGCCCCGCCCTATGGGGAGGCACGGGTGGCGAGGTCGGCCGTCCCATGCCCCGCCTGCTCCAGGTGCCGCCGTCCCCACAGGTGCCCGCCCCAGGCCCGCTGGAGCAGCCTGTGGCACGTGGGGTAAGTGGAGGCCGCGGCGGAGCCGTGCGTCCGACGGTTCTGGGGCGGGGGTCACCGGGGCCAGGCCCGCCAGGCCCTTACCTCACCAGGCCGCCTCCGCTGTCGCCCAGTCCCGGCCGCTGGCGGGAACTGACCTGGAGCAA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCTACCGCTGCCGGAAGAGCGCGGCGCCCGACGGAGCCGTGTGGAGGCCAAAACTCCTCCCGGAAGCCGCTACTGGCCCCGCTTGCCAGGCCCAGCGTCTTTTCTGCATAGGACCCGGGGGAAGCCGGGAAGCCGTTAGGGGGCGGGGCAAGCGGGCGGGCGTGCGTCGGTCAAGTTTCCCGGGTCGCGTCAGGCTTCTTTCTCAGCACCGAGCGAGCGACGTTCGTGAAGCTTTCGTTTTGAGCGGCCAGACTCTGCTGTTCTCAAGCCTGGGAGGCACAGAGAAGCAAGACTTCCGCA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAGAATCCCTCTATAATATCCTTGCACAGTTCCTCCGCAAGCACAATATGATTAATTTTTACTTGGCTGATGCCGAGATCTTAACTGCCCTAATCCTCGGCTCGAGTCGAGTTGGGGTTTCCTAAATGAATTACTATTGCAAATCTGGTTATGGCGCTCCACGAGATTATGGCAAAGGCTGAAGAATTTATTGACAGATGTGTTGTTAGACTGTGCTACGTTTTAGTGGCAGGTAGTGGAGCAGCCACCGGCGGACTCGAGAACGCAGTAACATGGCTCAAAAATGGTAGTATTTGTTTT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGGAACGCACCTGCGTCTAGACGCTGACGCCGTGCGCGGGGCGGGGGCCGGCGAGCATGCGCAGTGTGGTGTGGACAGGGCGTGGTTGGCAAGCATGCGCAATGTGGGCCGGGCGGGGCTGGCGAGCATGCGCAGTATGGGGCGGGGCGGGGCAGGGCTGGTGAGCAAGCCCGCGGGGAGCGAGCGCGGAGGCGGGCGTTGCTACGGCAACGGTGGGCGGCCAGGGAGTTGCCTGAGGGAAAAGGGGCAGACGTCCCTGGGTTCCGGTGTTCGCGGAGGAGTCGAGGCACGGAGAGGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTTCTGATTCTCGGTGAAAACGGTAAACATCGGCGTGCTATGTATGGGGATAGCAGGAGCTCTAGCATTTTTGCGCTGGGAAGATTATTCTTCTCTATCCGTCTATGCGAGCCGGCGTTTCGCAGGGGTACCCCCGGGGGCACTGCGTGGGAGGCCAGGAGGTTTTAGAATGGTCACCCGCCGGTTAGCTCCTTGGTAGTATTCTCGATGGACTGCGCGTTTGAGTCTTAAGGGCGTTTTCACGCTTCCGCCTTCTGGAAGCCTGCACTCCGTTTAGAGGAGCCAGAGACCCGTCCCGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTATTGGGGATGCAGGGCTCCGCGGTAGTTGACCTAGCAGGGCTAGTCACATCACGCCATAGGGAGCAGCAAGGGGCGCAGCAGTCAGCACGGCCGCTATTCAGATACGGTAACCAGACGCCGTGCGGCGTGCGCACCTGGCATCCAATGGCTTTGGGGATAGGGGCCGCCTGTGGGTTGTGGGTACGAATTAACGGAGTCGGAGCGGCGAATGAGCAGTCGCAAAGCCGAAGCCGGGCGTGGCGCTTTAGAGCGTCCCAGGCTCTCCTTCGGAAAGATGTCGGACACGGCAGTAGCTGA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCAGCGGCCGCGGCGACAGCTCCAGCTCCGGCTCCGGCTCCGGCTCCGGCTCCGGCTCCCGCGCCTGCCCCGCTCGGCCCAGCGCGCCCGGGCTCCGCGCCCCGACCCCGCCGCCGCGCCTGCCGGGGGCCTCGGGCGCCCCCGCCGCCCGCCTCACGCTGAAGTTCCTGGCCGTGCTGCTGGCCGCGGGCATGCTGGCGTTCCTCGGTGCCGTCATCTGCATCATCGCCAGCGTGCCCCTGGCGGCCAGCCCGGCGCGGGCGCTGCCCGGCGGCGCCGACAATGCTTCGGTCGCCTCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTTTTGAACATCTTGTGCCAGGATTTGACATTAAGCTCCAGTTCATCCGCACACCCCGCGGCCGCTTTCGCAAGGGACCGAGAGGGTGGCGGCCTCGCCCGGAGACTCGCGGCGCTGGGAGCTGCCTGCCCTCGGTGCGGGCTGCAGACCGGGCCGCGAGGCGGGCGCGCGGCGCGCACTAGGACCCAGCAGGGCTCCAGGCCGGGGTGGGAGCCGCGCCGCCGACCCCGGGCGGGCGCCGGGCGCAGGAGCCGGGGTTCCGGCCGCGATCTGCTGCAGCTCGGCCGGGAGACGGCGCGA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTGTGGCCAAAGAGAAGAAATGGGTTGAGACAGCAGGCCTGGCACTTACTTTACCTGGCCCAGTCTTGCCTGACAATTAAAAAAAGACGCTTTAGACTGGGCGCGGTGGCTCACGCCTGTAATCCTAGCACTTTGGGAGGCTGAGGCGGGCGGATCACGAGGCCAGGAGATTGAGACGATCCTGGCTAACACAGTGAAATCCCATCTCCACTAAAAATACAAAAACTTAGCCGGGCATGGTGGCGGGCGCCTGTAGTCCCAGCTACTCGGGCGGCTGAGGCAGGAGAATCGCTTGAACC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCATATAAACAGATTAACTGCATTCCCCAACGAGTAGAAATCTGCCCATCACTTTTGGGGTGGTTTGTTGCACCCACTATCGCCCAAGAAACTTCCCCAAGCCGGGTCATATTACAGGTCTCTAGCGCGCTGGCCGCACTCGCAGCCCACCATTCCCACAGAATGGTGTCATCGTAATCTGAAAAGTTGCCACTGGGCACCTGCGCAGGCTTGGCTGCGCCCTCTCGCGCCGCACGCTCCTTCCGGCGACGCGGATAGTCGTAAAGGTTTGAATACCTGATACAGGGAGAGAGGCCGAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCTTCACAGGGCACAGCCCCTGCAGAGCATCTTGGTCATTTGGAAGAGGACACGGTATCCCCTCTGGCCAGAGTATGTCAGAGAAGGAAGAGTAGGGCTTTTTTGTTTTGTTTTTTTTTAAAGGTGCTTGCTTGTTTAATGTAAATAATAGAAAGCCTTAATATCTTTTCTGTAACACGGAGTAATATTTTAATGTCATGTTTTGGATGTACATAATATATTTATAACAAAGCAGCAAGAGTCTACTTAACCTTGGCTGCCTCGTGGTGTTTCCTGGCTGGGTGGGGTGGGGGGTATCAA",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTATTCAGACCAGAACAAACTCGAGTGCCCCAGCTCCTCTGGCAATTAGGTAGCCAATAGTTGAGATTGCGTAGGCAACTACGTAAAATGCGCCGTGCCGTGCCCCTTTCTGCCACTCTGAGTTCGGTAACATAGCTCCGGTTGCTAGGTACTTAAGGTATCCAGTTCCCCCAGCGATTTGCATGAACCGAGAGGGAGTGTCTTCTGCCAACAGATCAGGCGGGGGATCATGAACTCATCCTCCCAGGGAATGCCGGTCGGGGATCCCCCGCGCAGCTCACAGGCCCTGGGAGTGAGCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACCTCGCTATTCTATGCACAGTTTTGAATCCTGCTTTTAAAATGCGATGTCTGCCTTGTCTTTAAATATTCTGAGGGGGAGGAGAGGAGATGTGTAATGCGACTGTCTATAATTTGAAACCGGACCTCACTATTTAGCGTCTCAGAAAACGCGGGGTTCATTGCCCTGGTCCCCCCCGCCTTCCCCCGCCCCCCGAACCTTCCGGCTGGCTCTCTTGCGGTCCCCGCCTCGGCGCTGATGTGGTCTGGCAGTGGAGATTGGCGCCCGGGCGGAGCACGATGGGCTTCCCCGCGGCAGGCA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCACGAGCGCCAGAGCCCCGCGCCTCCCCTAGCATATCCGGCGGAGTCCCTGCCCCGCAAGCGCCCACTATACGGCCGGACAACTGTTGTGGGCGTGGATGCAACCACGGGGGAGGGGGCAGGCGTGATCTCAGTCCCGCCCGTGCCCGAGCCCGCGCCCGAGCAGGGACTACATTTCCCTCCCCCCTGGGTCAGGGAGGAGTCTCCTCCTCTATCTATTAAGGATTCTCGCCTCCGCACATGCGTTATCCTTCCAAGGCCGAAAGGATTTGGAGTCCTGTACGCCAGAGGCCGAACTCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACCCTGCTGGGGGTACAGAGACGTTTCCGAAACTCAGCGGCTCGGTCGCCCCCAAAGGCCAGGCGGAACACACGCCCACACCGCGGCTCCCTTCCCGAAGTAAGACCGCCGGGCCACGGCCGCCCCCAGGAAGCCCCGCGCCCCGCACCCCACACCCGGGGACACAAACAGGCGCCGGGTAGCCGCGAGGGCCGGCGCGGCGCCTTTAAGAGGCGGCGGGCGGCGCTGCCCCCTGGCGGCCGCCCCGCCGCTTCCTCGCCGCCGCGGGCTCAAGCGGGGCGGCCGGGCCAGCGCGGGGCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCTTCACAGTCAGAACGCGGTACCGTTGTGCCCCAGCTCGACTGCAGCCTCCCCGCCCCGATACACTTGTCCCATCCTGTCCACGGCGCCCTTGTTTTGATTCGCTGCTGTCCCAGAGAGCTTAGTTATTCGTACGGGCAATCTTGACCCTTTCCCAGGACCTCCCCTAATACTGTTCATTGATCCTGGACCGGCAGCGAGACCCCTTCCGGCATCCAGAGGCCCAGTACTTTCAATCTCCTCCCCCAAAATGCTTTCCCTTTGTGGGGACCCTCGACATAGATGGACACCCAGCGGCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCGTTCTTGGCATCCTGAGAGGGCCAGTTAGTACTGAAGTCCCTTGGCTGCTCAAGGATTGCAGGGATGAGGCAAGTGGAACAGCCTCGGAACCTCCGAAAATGGGCACGCTCCAGGTCCCAGTTTCTATGGCAACCATACCGGCAAATTGGGCTCCGCAATGGTTTCTCCTGGAAAAACCGTGATTTTGGTTCCCGCGGACGTCTCTATGGTTTCGACAGCCTAGAAGGAACAAAACGGCATTTCCGGGAAGATGGCGGCGCACAAGTCAGGTCCGGCACATGTTTCCGCGGAGCGGAC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCTCTCGCGCCGCTCGCGTGACCGGCCGGTGTGTGCGCGAGGCCCCGGCTCCCGGGGCACGGACGGCCGGGCGCGCGCCTCTGCGAGGGGCGTCCGGGTCCGAGTCGGCGGTCCGGGCCGGCGCGAGGTGCGTGCGGGCGGGCCGCGGGGGTCCCGGACGGACACAAGCGCACACACTCCCGGAGGAGCCTTCGAGGCTGCTCTTCCTCGGCCAGACGGAGAGCGGCACTGTCTCCCCGCCCAGCGCTCACTCGCCCCGCGTCTCCCCCCGCGGCGGCTGCTCCTCCTCGGCACCGCCAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCCCTCCTCTGGTAGCCAGAATAAATTTTGCAGTAAGTTAGCAGAGTTGATTCCAAAAAGTCTGCACTGTGGCATCTCAGGAAACCAAACTTATCTGCTTAAGAAATAACTTGCAGAGAGAAAAGGAACTCTGACAAGGCTAATAACAGGGATTTCAAGCTGTGTTTCAGCAAGCCAGTCACAGAGTATTCACGTGTTAATTCACTGGCGTGTTCCGCGGCATAAGCACCCCCTCTCTGCCTTCCCCCAGTTCCAACTGTTGTGCTGCAGCAGATTTGGTCTGAGTCTGGGCAGAGCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATGATGCTGGGGACGGGACACAAACAGGCGTGACAGCAGCGACCAGGCCGGCGCGGCGCCCGTAGACTGAGTCGAGACACCCTGGGCACAGGCGGCCGCCCCGCCTCTGTGATAGTTACTGGGACGTGTGCCCAACGGCCGGGCCAGCGCGGGGCGGCGGCGGGCAGGGGCGGCGAGTGCCTCGACGTTCTAGTGCGCCCTTCTCGGCGGTCTTATGCACTATGGCAGCAAGAGTTTTGAATATGTTGCGGTAAGTTGAAATTATCGAGAATGCCAGGCGAGTTGGCCGCGTGCGCCCCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGCGACACGCCGTGCGCCTCCGCGGCTGCGCTACGAAAACGAGTCCCGGAGCGGCCCCGCGCCCGCCGCACCCGGCCCTCGCCCGCCCGAAGACAGGCGCCAAGCTGCCCCGCCGTCTCCCCAGCTAGCGCCCGGCCGCCGCCGCCTCGCGGGCCCCGGGCGGAAGGGGGCGGGGTCCCGATTCGCCCCGCCCCCGCGGAGGGATACGCGGCGCCGCGGCCCAAAACCCCCGGGCGAGGCGGCCGGGGCGGGTGAGGCGCTCCGCCTGCTGCGCGTCTACGCGGTCCCCGCGGGCCTTCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTAGCCAGGAGGTAGCATCTGTCTCCAGGTGCCCGTGGGGTGTCCTCCATGAGGACCACCTCTTCTAACCACCTGCCCAGCTCAGAGGACCAGCTGGCAGGAAAGACGCTGCACCGCCCATGAGGCCAGAGCTACAAGTTGCTTCTAGGAGTGGCTGTGGGCGGAGGCTGGTGGTTCTGAAGGTGGCGGTGGTTCTGCAGCGTGGCTCCCCACAGCCCCTTTCATCTGAACAGTAGGGCTCATGCCACTTGTTAATCAGCCTTCCAGTGGCACGAGGAGTCATTTTCTGGGGATAGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCCTCCTCCCACGCTGCGCGCGCACCTCCCCGCCCCCACCCCTACCCGCTGGCGTGCCCAGTGGAACGGAGCCTTGTGTCTCCGCCTCAAGTCCCCGGATGCTCACCTCCCCGACTCGCCCCCGCTGTGGCCCCGCCCCCGCGCGGCTCTTCGTGCCACGTCACCGCCTGCGTCGCTTCCGGAGGCGCAGCGGGCGATGACGTAGAGGGACGTGCCCTCTATATGAGGTTGGGGAGCGGCTGAGTCGGCCTTTTCCGCCCGCTCCCCCCTCCCCCCGAGCGCCGCTCCGGCTGCACCGCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCATGCAGCGACGGCCGCCGCGGAGCTCCGAGCAGCGGTAGCGCCCCCCTGTAAAGCGGTTCGCTATGCCGGGGCCACTGTGAACCCTGCCGCCTGCCGGAACACTCTTCGCTCCGGACCAGCTCAGCCTCTGATAAGCTGGACTCGGCACGCCCGCAACAAGCACCGAGGAGTTAAGAGAGCCGCAAGCGCAGGGAAGGCCTCCCCGCACGGGTGGGGGAAAGCGGCCGGTGCAGCGCGGGGACAGGCACTCGGGCTGGCACTGGCTGCTAGGGATGTCGTCCTGGATAAGGTGGCATG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTGAGCCAGGCTGAAGCGGGATAGGAAGGGTAAAGGTGCTAGCTGAGCCGTAGTACGGCCCAGCAGTTCTCCAGGGGACAAGCTAGGCCCCCGCTGGAGGGAGTGGTGAGACTCAGCCGGGGCCTGATTCTGGTCAGCACCTTGGACAGCAGCCAGTGCCCTCCCTGGCCCCGCCTGGCCCCGCCTGGCCCCACATCCGCTACTCCTGGGCACCTCCTCAAATAGCACAGCCTCCAGCTGGTGCCCAGGGCCTGAATACACAGAGCGCTGAGAGAGTGGGGCAGTGTGGTCACGGAC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCTGGGAAAAGCTGAGAGCATAATCGAGCGGCAAAGAAAACGCGACCACTGAGGACACTTAGAAGGGGGCGGGGCCTTTTTGTGGTACTAGGAGCTCGAGAAATTGACTAGATTACGAGTGGTTTACGACTCAAGCATACAACTTACAGCTTGGGCCTTCCGTAGCGGGCGTTGCGGGGGAACGTACAGTGCTTGAGGGGCGAGGTTTGCCGGAGCCTCATTCTAAAGGAATCTTCTAGTTCCAAGCGTTGTTCCAATTGGGCCGGAGCGTGCAACGGGACCCACGGAACTACAGGTGTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAAGCCACCTCCGCGCCGGCAGCTCCGTTTCTAGGGAAACAGGACCAATTCCAGAAGTTGGGGTGTTACGTGTGCGATATACCTGCCCAAGAGCTGGTATCAATAGAGCGGGCGGCATGGTAGTGTTTCCCCCGGCCATGGTAAAATGCGAACCAGATCCTCGGCGGGTAACCGCGCCCCTAGGCGTTGGCATGTGTCGGGGTAGCATGACGGGGTCGTCCAGTTCTTAGCCAAGATGAATCCGGCTCCGAAGGTCCCCGGCTTCGTGGCCATAAAGGCGAACACCGCTGGAAAGGTGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGGCTCGGGGACGGGACGGGCGAGCAGTCATTTCTGAATAGCGCGTGCAGAGGGTCATCGCGAAAAGACTAAGACGCGACATAATGGACTGACTCGGGGCATGGCGGGGTGGAGGCGGGAAAGTGTCGAGGCCGGCTAATGCCTGGACAGGGAGGTGGGCATCGCCAGGGCGACGATGAGGGTCGCCACTCCGGAACACGATTAGCATCAGTGCACTGACCTCTGCCCCCACGACTCTCCAACACGCCCGATGATGTTGCGTGCGGCAGCTGTCGGCTAGGAGTCGCGGGGTCTGTGCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATGGTATTTCAAGGTCATAAATTACAAATGTGCATATTTATTATTCCCCTATAAAATGTTAATTAAAAAATTTTCATATTTACTTGATGTAACAGAAATGAAACCTGTAATGAGGGATCTAATTTCTTTTTCTTTTTTCATTCCAGAATGTATGACAACATGTCCACAATGGTGTACATAAAGGAAGACAAGTTGGAGAAGCTTACACAGGATGAAATTATTTCTAAGACAAAGCAAGTAATTCAGGGGCTGGAAGCTTTGAAGAATGAGCACAATTCCATTTTACAAAGTTTGCTGGA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACGACCCTGGCCTCCGACTTCAACGACTTCATAAGGCGGCGTTTCTGGGCGCAGCCGTGTCGCTCCTGGTGAGAGGCCGCCGGCAGGCGGGATCCAGCGCCCTCCGGGGCACCGCGGGCGAGACCGTCGCCTTCGCACCCCCGGCGCGGTCGACCCCGCGGGGCCGTCGGGTCCTGGGTTCCCCGCCGCGTTGCGCTCGTCCCCCTCCTGTCAGAACCTGGGCCCCCGCCCCGCCCACCGGCGCGGGGCCTCTCCTCCTCCCGCTAACGGGCGGTCGGCCGCCTCCTTCCTCTCTTGGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTTTCTGAGGCTGAGGAGGTACCCAGGTAAGCACACTCTGGGTTCTCTGGTGAGCACACACCGAGCTGCCTGCATCTCCCCTACCCATGAAGGCAGCTAGCCGCCGGGCCAGTGGTGGGAAGGGGAGGCGGCCATTAGCGCCGTTCCCGCCGGGACTTGAAGCGCCCGGCCGCGGCAAGCCCCGCCCTCGGCGCGCCCCCGCGTCCGCGCGCGCTCCTCGGGTCTGCGCGGAGCCGGCGTCGGCGCGCGCTTGGGCGCCTGGCGAAGACCGAGAGAGGCTGGCGGGATCTCAGCGGCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCGCCCGTTGGACATACCTCTAAATTGGGAGCGCGGTGCCGATTTACGTGACCGCCGTGGTTCGCACGGTGGGTAAACAGTGGATCGTGGGGTGGTGAGAAGGTTCCGATGGGTGTTCTTACTACGGCCCGATCCTAGCAGTCGCCGACTTTGTTATTCTTATTGGTGGATGGGGGGCCGCTGATATAAATCTGGGAGATGAGGAGATAGGCGGGAGCGAGGGCTGACTCTGCGTGTCGAAGGAGGTAGAGAAAAAGCGTTGCTTAAAGGGAGGGAGCGTAGCTTGGTTGCTCCGTAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCTCTCTTCGGGCGGCAAGTGCCATACGTATAGATAAGCGGAATTTACATGGTGTCACAGTTAGCGCAAGGTCGAATAAATGCGGCTTAACACGACAGGTATCTCCAAATAAGTGTGTCTTACGGCGTAGATCGCGAGTTGAGCTGTCATTTCTTTCCAAGCGGCGCAGGACAGGACCCAGAAGCCAGAGCGCAGGAACTGCTTCCGGTTTGAGTTTTTATTAGAATAACGGGCTGTGGGAGAGGCAAGTGGCGGATCTTCAGGTCGGCGAGGGTGTGACATTCGACCCACAGGCCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCATATTCTGTCTGAAATCGTGTGCACCGAAATCCCCGCCTTGCGGTGGAGGCTGGCGCTAGGCGGCCTCAGCCTCGGCCTGCTGCGCTCAGGAACCCGCGCCCCGGCTCCTCGGCGATCCATTGCTCTTTCCTCTGGCGCCGGCCGCAGGCCTCGGTCACGCCCCCAGCGGCCCGTTGGTTTCCGGGTCCCGCGGGGTGCCCCCGCCCACACGCTATGCCTTAAATTGGGCCAGGCTGAGGCGCTGCTGCTGGAGCGGCCGATCCGAGACGTGGCTCCCTGGGCGGCAGAACCATGTTG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGGTCTTGTGTCTCGGCGTACCATATCTTTTGGGTTCCCCGGCTGCCCCGGAGGTTCGCGGCGAGAGGACGGGCCGGGTGCGGCTAGTTTACCTGAGCAGTGTGAACTACCTCACTCCCATCGGGACCAGCCGGCATAAACACTCTTATTGCCTGTCAACGTTTTCGAGACATATTTTGTTCGCGAGCCCGCAGACAACGCGCATTCTGGAGTTCGAACGCTCAGTTCGCGTCCCCGAGAAGATATTATTCAGATGCCTGCGAGCTGGGGCCGGGTGGCCGCGCCGGGTCGGAGTGTGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCGGAGACAGGTGGGAGGGTGGGGGATGCCGGCAGGAGAGGGAAGCAGGACCGTGCGCGCCGCCACCCACCCGCAGTCCGTCCGCCCCCGGGCGCCCAGCGCGTGGCCCGAGCGGCGCTGCCCGCCGAGCCGGGAGCCCCCTCGGCCCCTCCCTTCAGGCGCGGGCCGGGGGCGGCTTCTCCGCGACCTTATGTAACCGGGCGGGAGGGGCCGGGCGGGCATGGGCCTTCCCGGCCCGGAGCTGGGAGTCGAAGGGGCGGGAGGCGTGATGGTGAACTCGCAAGAAGTTTGAGGGACG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCAGGACTGAGTCACCTCCTGAAGGCCCCGTTCCAAAGCACTCCCACGGGGGATTAGGTTTCAACACAGGAATTTTTGGCGACACAAACATTGAGACCACAGCACCCAGCTCTCTGGGTTCCCAGAACCCTGTGGGAGGGCTGGGTCGCGCCTGCCCCCTTTTAAACACGAGGCTTCCATGGAAGCTTCTTGCCGGAGGCTGGACACTGAGGAATGGACAGGATCAACACTCTTCCACCGGCTCTTCCACCTCCCGATCACTTCCACCATCGCTGGGGGAAGAACGAATGCACCGCGTGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACAGCCCCAGATGGTGACTACTGGAGACTGCTGCCCCCAGGTATCCACATTGTCATTGCCCAAGCCCCTGGCTACGCCAAAGTCATCAAGAAAGTCATCATCCCCGCCCGGATGAAGAGGGCTGGCCGTGTGGACTTCATTCTGCAACCTCTGGGGATGGGACCCAAGAACTTTATTCATGGGCTGCGGAGGACTGGGCCCCACGACCCACTGGGAGGTGCCAGCTCTTTGGGGGAGGCCACGGAGCCCGACCCGCTCCGGGCGCGCAGGCAGCCCTCGGCCGACGGGAGTAAGCCCTG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AATCGTGTAATTCCCGCTTCCTATGGCAAGCGGCGCCCAGTTCTGGGTGACTTGTGAAATGAGCGTCCTGCTAGGCCATAGTCTCTACGAGGAGGATATAATGTAAAGTCTGGAGCGAACGTAAATGCCGTGACGTGTATTTGTGAGACTTCGCCCGGTCGGTGAGCTGGGTGCATAGGTTCCGAGCGCGCGACCACTTAGCTTCCCTTCTACAACAACCACCTGTGTGCGCATGCCCTCTCATCTTACAGTTGTCAAAGTGCTTTTGCCAGAGCGGTCACATAATCAACCTGAAAGAAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCTTAGAATATTGGGCATACAATTTACTTTTATAAATGTCTATGAAATAAAATTCTTATATAATTATATAAAATTACATAATATATAATTATATTGAAATAATTATAACTGTTGAATAAATGTATTATTTTTCCTTTGAACTATTTAACAGGGTTATATAACAAACTCATCACGAGTTGTGTCTAATAAGTCATCAGAGTTACTGTTTGACTTGACCCAGGATACAGGATTATCACATTACCAAGGGGGACCAACACTTTCTATGGCAGGTTGGTTTTAGTATTTTTTTCATAAAGGTT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGCGCGCGCGCCGGGCCGAGGGGCCGAGAGGCGGGGTCTTACAGCGACCGCGGGAAGAGGGCGCCCCAGAGCTGAGCCGGAGGCCGGCTGCCAGCGGGGCGAGGGTGGGCGCGAGCGCAGGGGCGGGCCCCGAGGGAGGGCGGGGACGGTGAGGGGGCGGGGTCGGGCTAGGCGGGGACGCGCTCGCGGGGTGGGGAAGGCGGGGGCGCGGCGGTGGCGGGAGCGTGCCCGGTCCCCGCCCCTGTTCCCACTCTCCTTCCACCTCGGACCGGCCGGGGCTCCGCAGAGCCAAAGCTCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATTTCCGATTCCTTGACTAACGTACGGGGGCCCAGCACTTTCATTCCATCCCATCACCCCATGATCCTTCCCCAGACTCTAATACTGGCCGTAATATTTACCCCCCCCTGCTCCATGTGCGTAGATCAATGACACGAGCAAGCCATTGGTCTAAAACCCAGGGTTCTCTCGCAGAGGCCCTCCATTGTCTCTGCGGTGTCACTTCGCGCACACACATCAGAGCGGGCCTCTCTGTGAGGCGCGGGGTTGATGTACTCACACGCTGCGTTGATCGGAGGTCGCTGCCAAGCATGGCGCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCAGCAAAGGCTTCCCTGAGGAATTGGGAATAGCCAGCCATACGGGTGGAGCCGGTCAACTCCTTGTGGTTGGGAGCTAACCGATAGCCCCTACAAGGAGCATTCCAACTATGTTATATAACATTGTGATTTTCGCGAAGGCAACGACTTATTCTTAAGACAGTATCTATCTGAATCTGAAGATCCTTGGTCTGGAGATTGGTAAAAATCTCCAGTCATCTTATGTTTCTATTATACTCTACTGGCGTTTTTACGTGACGCGGGGAGTCTACCTCCCCCAACCATAATTTAACAGTAA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    }
+]

04-gene-sft/.ipynb_checkpoints/merge_llama_with_dna_lora-checkpoint.py

@@ -0,0 +1,368 @@
+"""
+Usage: 
+python merge_llama_with_chinese_lora.py \
+    --base_model path/to/llama/model \
+    --lora_model path/to/first/lora/model [path/to/second/lora/model] \
+    --output_type [pth|huggingface] \
+    --output_dir path/to/output/dir
+"""
+import subprocess
+import os
+# 设置环境变量, autodl一般区域
+result = subprocess.run('bash -c "source /etc/network_turbo && env | grep proxy"', shell=True, capture_output=True, text=True)
+output = result.stdout
+for line in output.splitlines():
+    if '=' in line:
+        var, value = line.split('=', 1)
+        os.environ[var] = value
+        
+import argparse
+import json
+import os
+import gc
+import torch
+import peft
+from peft import PeftModel
+from transformers import LlamaForCausalLM, LlamaTokenizer
+from huggingface_hub import hf_hub_download
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--base_model', default=None, required=True,
+                    type=str, help="Please specify a base_model")
+parser.add_argument('--lora_model', default=None, required=True,
+                    type=str, help="Please specify LoRA models to be merged (ordered); use commas to separate multiple LoRA models.")
+parser.add_argument('--offload_dir', default=None, type=str,
+                    help="(Optional) Please specify a temp folder for offloading (useful for low-RAM machines). Default None (disable offload).")
+parser.add_argument('--output_type', default='pth',choices=['pth','huggingface'], type=str,
+                    help="save the merged model in pth or huggingface format.")
+parser.add_argument('--output_dir', default='./', type=str)
+
+
+emb_to_model_size = {
+    4096 : '7B',
+    5120 : '13B',
+    6656 : '33B',
+    8192 : '65B',
+}
+num_shards_of_models = {'7B': 1, '13B': 2, '33B': 4, '65B': 8}
+params_of_models = {
+    '7B':
+        {
+        "dim": 4096,
+        "multiple_of": 256,
+        "n_heads": 32,
+        "n_layers": 32,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '13B':
+        {
+        "dim": 5120,
+        "multiple_of": 256,
+        "n_heads": 40,
+        "n_layers": 40,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '33B':
+        {
+        "dim": 6656,
+        "multiple_of": 256,
+        "n_heads": 52,
+        "n_layers": 60,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '65B':
+        {
+        "dim": 8192,
+        "multiple_of": 256,
+        "n_heads": 64,
+        "n_layers": 80,
+        "norm_eps": 1e-05,
+        "vocab_size": -1,
+        },
+}
+
+def transpose(weight, fan_in_fan_out):
+    return weight.T if fan_in_fan_out else weight
+
+# Borrowed and modified from https://github.com/tloen/alpaca-lora
+def translate_state_dict_key(k):
+    k = k.replace("base_model.model.", "")
+    if k == "model.embed_tokens.weight":
+        return "tok_embeddings.weight"
+    elif k == "model.norm.weight":
+        return "norm.weight"
+    elif k == "lm_head.weight":
+        return "output.weight"
+    elif k.startswith("model.layers."):
+        layer = k.split(".")[2]
+        if k.endswith(".self_attn.q_proj.weight"):
+            return f"layers.{layer}.attention.wq.weight"
+        elif k.endswith(".self_attn.k_proj.weight"):
+            return f"layers.{layer}.attention.wk.weight"
+        elif k.endswith(".self_attn.v_proj.weight"):
+            return f"layers.{layer}.attention.wv.weight"
+        elif k.endswith(".self_attn.o_proj.weight"):
+            return f"layers.{layer}.attention.wo.weight"
+        elif k.endswith(".mlp.gate_proj.weight"):
+            return f"layers.{layer}.feed_forward.w1.weight"
+        elif k.endswith(".mlp.down_proj.weight"):
+            return f"layers.{layer}.feed_forward.w2.weight"
+        elif k.endswith(".mlp.up_proj.weight"):
+            return f"layers.{layer}.feed_forward.w3.weight"
+        elif k.endswith(".input_layernorm.weight"):
+            return f"layers.{layer}.attention_norm.weight"
+        elif k.endswith(".post_attention_layernorm.weight"):
+            return f"layers.{layer}.ffn_norm.weight"
+        elif k.endswith("rotary_emb.inv_freq") or "lora" in k:
+            return None
+        else:
+            print(layer, k)
+            raise NotImplementedError
+    else:
+        print(k)
+        raise NotImplementedError
+
+
+def unpermute(w):
+    return (
+        w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)
+    )
+
+
+def save_shards(model_sd, num_shards: int):
+    # Add the no_grad context manager
+    with torch.no_grad():
+        if num_shards == 1:
+            new_state_dict = {}
+            for k, v in model_sd.items():
+                new_k = translate_state_dict_key(k)
+                if new_k is not None:
+                    if "wq" in new_k or "wk" in new_k:
+                        new_state_dict[new_k] = unpermute(v)
+                    else:
+                        new_state_dict[new_k] = v
+
+            os.makedirs(output_dir, exist_ok=True)
+            print(f"Saving shard 1 of {num_shards} into {output_dir}/consolidated.00.pth")
+            torch.save(new_state_dict, output_dir + "/consolidated.00.pth")
+            with open(output_dir + "/params.json", "w") as f:
+                json.dump(params, f)
+        else:
+            new_state_dicts = [dict() for _ in range(num_shards)]
+            for k in list(model_sd.keys()):
+                v = model_sd[k]
+                new_k = translate_state_dict_key(k)
+                if new_k is not None:
+                    if new_k=='tok_embeddings.weight':
+                        print(f"Processing {new_k}")
+                        assert v.size(1)%num_shards==0
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+                    elif new_k=='output.weight':
+                        print(f"Processing {new_k}")
+                        if v.size(0)%num_shards==0:
+                            splits = v.split(v.size(0)//num_shards,dim=0)
+                        else:
+                            size_list = [v.size(0)//num_shards] * num_shards
+                            size_list[-1] += v.size(0)%num_shards
+                            splits = v.split(size_list, dim=0) # 13B: size_list == [24976,24977]
+                    elif new_k=='norm.weight':
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+                    elif 'ffn_norm.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+                    elif 'attention_norm.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+
+
+                    elif 'w1.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+                    elif 'w2.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+                    elif 'w3.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+
+
+                    elif 'wo.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+
+                    elif 'wv.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+
+                    elif "wq.weight" in new_k or "wk.weight" in new_k:
+                        print(f"Processing {new_k}")
+                        v = unpermute(v)
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+                    else:
+                        print(f"Unexpected key {new_k}")
+                        raise ValueError
+                    for sd,split in zip(new_state_dicts,splits):
+                        sd[new_k] = split.clone()
+                        del split
+                    del splits
+                del model_sd[k],v
+                gc.collect()    # Effectively enforce garbage collection
+
+            os.makedirs(output_dir, exist_ok=True)
+            for i,new_state_dict in enumerate(new_state_dicts):
+                print(f"Saving shard {i+1} of {num_shards} into {output_dir}/consolidated.0{i}.pth")
+                torch.save(new_state_dict, output_dir + f"/consolidated.0{i}.pth")
+            with open(output_dir + "/params.json", "w") as f:
+                print(f"Saving params.json into {output_dir}/params.json")
+                json.dump(params, f)
+
+
+if __name__=='__main__':
+
+    args = parser.parse_args()
+    base_model_path = args.base_model
+    lora_model_paths = [s.strip() for s in args.lora_model.split(',') if len(s.strip())!=0]
+    output_dir = args.output_dir
+    output_type = args.output_type
+    offload_dir = args.offload_dir
+
+    print(f"Base model: {base_model_path}")
+    print(f"LoRA model(s) {lora_model_paths}:")
+
+    if offload_dir is not None:
+        # Load with offloading, which is useful for low-RAM machines.
+        # Note that if you have enough RAM, please use original method instead, as it is faster.
+        base_model = LlamaForCausalLM.from_pretrained(
+            base_model_path,
+            load_in_8bit=False,
+            torch_dtype=torch.float16,
+            offload_folder=offload_dir,
+            offload_state_dict=True,
+            low_cpu_mem_usage=True,
+            device_map={"": "cpu"},
+        )
+    else:
+        # Original method without offloading
+        base_model = LlamaForCausalLM.from_pretrained(
+            base_model_path,
+            load_in_8bit=False,
+            torch_dtype=torch.float16,
+            device_map={"": "cpu"},
+            cache_dir=None,  # 不使用缓存目录
+            force_download=False,  # 禁止从远程下载
+            local_files_only=True  # 强制仅从本地文件加载                                    
+        )
+
+    ## infer the model size from the checkpoint
+    embedding_size = base_model.get_input_embeddings().weight.size(1)
+    model_size = emb_to_model_size[embedding_size]
+    print(f"Peft version: {peft.__version__}")
+    print(f"Loading LoRA for {model_size} model")
+
+    lora_model = None
+    lora_model_sd = None
+    for lora_index, lora_model_path in enumerate(lora_model_paths):
+        print(f"Loading LoRA {lora_model_path}...")
+        tokenizer = LlamaTokenizer.from_pretrained(lora_model_path, 
+                                                   cache_dir=None,  # 不使用缓存目录
+                                                   force_download=False,  # 禁止从远程下载
+                                                   local_files_only=True  # 强制仅从本地文件加载
+                                                  )
+        
+        print(f"base_model vocab size: {base_model.get_input_embeddings().weight.size(0)}")
+        print(f"tokenizer vocab size: {len(tokenizer)}")
+
+        model_vocab_size = base_model.get_input_embeddings().weight.size(0)
+        assert len(tokenizer) >= model_vocab_size, \
+        (f"The vocab size of the tokenizer {len(tokenizer)} is smaller than the vocab size of the base model {model_vocab_size}\n"
+        "This is not the intended use. Please check your model and tokenizer.")
+        if model_vocab_size != len(tokenizer):
+            base_model.resize_token_embeddings(len(tokenizer))
+            print(f"Extended vocabulary size to {len(tokenizer)}")
+
+        first_weight = base_model.model.layers[0].self_attn.q_proj.weight
+        first_weight_old = first_weight.clone()
+
+        print(f"Loading LoRA weights")
+        if hasattr(peft.LoraModel,'merge_and_unload'):
+            try:
+                lora_model = PeftModel.from_pretrained(
+                    base_model,
+                    lora_model_path,
+                    device_map={"": "cpu"},
+                    torch_dtype=torch.float16,
+                    local_files_only=True
+                )
+            except RuntimeError as e:
+                if '[49953, 4096]' in str(e):
+                    print("The vocab size of the tokenizer does not match the vocab size of the LoRA weight. \n"
+                           "Did you misuse the LLaMA tokenizer with the Alpaca-LoRA weight?\n"
+                           "Make sure that you use LLaMA tokenizer with the LLaMA-LoRA weight and Alpaca tokenizer with the Alpaca-LoRA weight!")
+                raise e
+            assert torch.allclose(first_weight_old, first_weight)
+            print(f"Merging with merge_and_unload...")
+            base_model = lora_model.merge_and_unload()
+        else:
+            base_model_sd = base_model.state_dict()
+            try:
+                lora_model_sd = torch.load(os.path.join(lora_model_path,'adapter_model.bin'),map_location='cpu')
+            except FileNotFoundError:
+                print("Cannot find lora model on the disk. Downloading lora model from hub...")
+                filename = hf_hub_download(repo_id=lora_model_path,filename='adapter_model.bin')
+                lora_model_sd = torch.load(filename,map_location='cpu')
+            if 'base_model.model.model.embed_tokens.weight' in lora_model_sd:
+                assert lora_model_sd['base_model.model.model.embed_tokens.weight'].shape[0]==len(tokenizer), \
+                ("The vocab size of the tokenizer does not match the vocab size of the LoRA weight. \n"
+                "Did you misuse the LLaMA tokenizer with the Alpaca-LoRA weight?\n"
+                "Make sure that you use LLaMA tokenizer with the LLaMA-LoRA weight and Alpaca tokenizer with the Alpaca-LoRA weight!")
+
+            lora_config = peft.LoraConfig.from_pretrained(lora_model_path)
+            lora_scaling = lora_config.lora_alpha / lora_config.r
+            fan_in_fan_out = lora_config.fan_in_fan_out
+            lora_keys = [k for k in lora_model_sd if 'lora_A' in k]
+            non_lora_keys = [k for k in lora_model_sd if not 'lora_' in k]
+
+            for k in non_lora_keys:
+                print(f"merging {k}")
+                original_k = k.replace('base_model.model.','')
+                base_model_sd[original_k].copy_(lora_model_sd[k])
+
+            for k in lora_keys:
+                print(f"merging {k}")
+                original_key = k.replace('.lora_A','').replace('base_model.model.','')
+                assert original_key in base_model_sd
+                lora_a_key = k
+                lora_b_key = k.replace('lora_A','lora_B')
+                base_model_sd[original_key] += (
+                    transpose(lora_model_sd[lora_b_key].float() @ lora_model_sd[lora_a_key].float(),fan_in_fan_out) * lora_scaling
+                )
+                assert base_model_sd[original_key].dtype == torch.float16
+
+        # did we do anything?
+        assert not torch.allclose(first_weight_old, first_weight)
+
+    tokenizer.save_pretrained(output_dir)
+
+    if output_type=='huggingface':
+        print("Saving to Hugging Face format...")
+        LlamaForCausalLM.save_pretrained(base_model, output_dir) #, state_dict=deloreanized_sd)
+    else: # output_type=='pth
+        print("Saving to pth format...")
+
+        base_model_sd = base_model.state_dict()
+        del lora_model, base_model, lora_model_sd
+
+        params = params_of_models[model_size]
+        num_shards = num_shards_of_models[model_size]
+        n_layers = params["n_layers"]
+        n_heads = params["n_heads"]
+        dim = params["dim"]
+        dims_per_head = dim // n_heads
+        base = 10000.0
+        inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))
+
+        save_shards(model_sd=base_model_sd, num_shards=num_shards)

04-gene-sft/.ipynb_checkpoints/pip_list-checkpoint.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828ede2bc2aeec77ad37dd1e5ba9ff2fbbb481dbe3c2729b6cdafc4feb7ac669
+size 7419

04-gene-sft/.ipynb_checkpoints/run_clm_pt_with_peft-checkpoint.py

@@ -21,6 +21,17 @@ https://huggingface.co/models?filter=text-generation
 """
 # You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
 
+import subprocess
+import os
+# 设置环境变量, autodl一般区域
+result = subprocess.run('bash -c "source /etc/network_turbo && env | grep proxy"', shell=True, capture_output=True, text=True)
+output = result.stdout
+for line in output.splitlines():
+    if '=' in line:
+        var, value = line.split('=', 1)
+        os.environ[var] = value
+        
+
 import logging
 import numpy as np
 import math

04-gene-sft/.ipynb_checkpoints/run_pt-checkpoint.sh

@@ -6,17 +6,17 @@ modules_to_save="embed_tokens,lm_head"
 lora_dropout=0.05
 
 pretrained_model=./llama-7b-hf
-dna_eng_tokenizer_path=./merged_dna_eng_tokenizer_hf
+dna_eng_tokenizer_path=./merged_gene_eng_tokenizer_hf
 dataset_dir=./train_data
 data_cache=temp_data_cache_dir
-per_device_train_batch_size=32
-per_device_eval_batch_size=32
+per_device_train_batch_size=4
+per_device_eval_batch_size=4
 gradient_accumulation_steps=8
 output_dir=dnahlm_llama_7b
 
 deepspeed_config_file=ds_zero2_no_offload.json
 
-torchrun --nnodes 1 --nproc_per_node 6 run_clm_pt_with_peft.py \
+torchrun --nnodes 1 --nproc_per_node 10 run_clm_pt_with_peft.py \
     --deepspeed ${deepspeed_config_file} \
     --model_name_or_path ${pretrained_model} \
     --tokenizer_name_or_path ${dna_eng_tokenizer_path} \
@@ -39,8 +39,8 @@ torchrun --nnodes 1 --nproc_per_node 6 run_clm_pt_with_peft.py \
     --save_total_limit 3 \
     --save_steps 200 \
     --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --preprocessing_num_workers 128 \
-    --block_size 512 \
+    --preprocessing_num_workers 64 \
+    --block_size 256 \
     --output_dir ${output_dir} \
     --overwrite_output_dir \
     --ddp_timeout 30000 \

04-gene-sft/.ipynb_checkpoints/run_sft-checkpoint.sh

@@ -8,8 +8,8 @@ lora_dropout=0.05
 pretrained_model=dnahlm-merge-hf
 chinese_tokenizer_path=dnahlm-merge-hf
 dataset_dir=sft_data
-per_device_train_batch_size=32
-per_device_eval_batch_size=32
+per_device_train_batch_size=8
+per_device_eval_batch_size=8
 gradient_accumulation_steps=8
 output_dir=dnahlm-llama7b-sft
 #peft_model=peft_model/dir
@@ -29,7 +29,7 @@ torchrun --nnodes 1 --nproc_per_node 6 run_clm_sft_with_peft.py \
     --do_eval \
     --seed $RANDOM \
     --fp16 \
-    --num_train_epochs 8 \
+    --num_train_epochs 3 \
     --lr_scheduler_type cosine \
     --learning_rate ${lr} \
     --warmup_ratio 0.03 \
@@ -43,7 +43,7 @@ torchrun --nnodes 1 --nproc_per_node 6 run_clm_sft_with_peft.py \
     --save_steps 200 \
     --gradient_accumulation_steps ${gradient_accumulation_steps} \
     --preprocessing_num_workers 4 \
-    --max_seq_length 512 \
+    --max_seq_length 256 \
     --output_dir ${output_dir} \
     --overwrite_output_dir \
     --ddp_timeout 30000 \

04-gene-sft/2-gpt2-instruction-ft.ipynb

@@ -127,20 +127,76 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "dc04d5e3-7623-4d59-9f3b-ad03e339db11",
+   "execution_count": 1,
+   "id": "b28e3499-bbff-4548-9f85-2baee088cabf",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dc04d5e3-7623-4d59-9f3b-ad03e339db11",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['sequence', 'label'],\n",
+       "        num_rows: 59195\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from datasets import load_dataset\n",
     "# 1. load ~11k samples from promoters prediction dataset\n",
-    "dataset = load_dataset(\"dnagpt/dna_promoter_300\")\n",
-    "dataset"
+    "dna_dataset = load_dataset(\"dnagpt/dna_promoter_300\")\n",
+    "dna_dataset"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
+   "id": "f7332fa1-3343-4247-b4cc-54733dff6964",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'sequence': 'TAAATACGGAAGTTTATTACTTGAGGAATAGATGGAATCGTCGGGCGTGAGAGATCATAATCGGCTGCTTCTGGGAGCCGCACGTGGGAAAGACTTATCCCCGACGGAGCTGGGACTGGGGCACAAACCGGAAGGAACACATCTGACCGAGAAAGAGACCAAGTGGCTCAGGTAGGACCAAAGCGAGCAAGGCTGCGGGTCCTGTTGCTCTCTGTCCTGTAAATTTAAACGTTACGCCACCTGGTAATGATACCCTCGTCCTCCGAGGCGACAAGTCAGAACTTCCACCAAGGGCATTAC',\n",
+       " 'label': 0}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dna_dataset[\"train\"][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "id": "93d09d8d-f521-49f7-b0e0-7ac089dfbf49",
    "metadata": {},
    "outputs": [],
@@ -169,10 +225,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "9f9c0e5a-6591-47ac-b358-d746a00dfc0a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.', 'input': 'TAAATACGGAAGTTTATTACTTGAGGAATAGATGGAATCGTCGGGCGTGAGAGATCATAATCGGCTGCTTCTGGGAGCCGCACGTGGGAAAGACTTATCCCCGACGGAGCTGGGACTGGGGCACAAACCGGAAGGAACACATCTGACCGAGAAAGAGACCAAGTGGCTCAGGTAGGACCAAAGCGAGCAAGGCTGCGGGTCCTGTTGCTCTCTGTCCTGTAAATTTAAACGTTACGCCACCTGGTAATGATACCCTCGTCCTCCGAGGCGACAAGTCAGAACTTCCACCAAGGGCATTAC', 'output': 'Non-promoter'}\n"
+     ]
+    }
+   ],
    "source": [
     "example = dna_dataset[\"train\"][0]\n",
     "print(build_prompt(example))"
@@ -180,7 +244,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "83070a23-1604-4d28-b371-e01060331ed5",
    "metadata": {},
    "outputs": [],
@@ -199,10 +263,40 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "89fb8ed3-aa58-462f-b2a6-ce445c597a33",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7b84d3a64bf645ada13d0cada2d9f524",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split: 0 examples [00:00, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 59195\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "dna_ft_dataset = load_dataset(\"json\", data_files='data/dna_promoter_300.jsonl')\n",
     "dna_ft_dataset"
@@ -210,10 +304,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "e4f7b75f-6ccb-4fda-8004-40df7d52678f",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 53275\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 5920\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.9, seed=42)\n",
     "data"
@@ -221,7 +335,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "36d9ee0e-8423-4529-aa7e-fda2728fab2f",
    "metadata": {},
    "outputs": [],
@@ -235,13 +349,14 @@
     "from tokenizers import Tokenizer\n",
     "from transformers import GPT2TokenizerFast\n",
     "\n",
+    "#需要使用生物序列+英文的多模态大模型\n",
     "tokenizer = GPT2Tokenizer.from_pretrained(\"dnagpt/gene_eng_gpt2_v0\")\n",
     "tokenizer.pad_token = tokenizer.eos_token"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "id": "871baee0-f06f-4422-a741-af533f7d92e1",
    "metadata": {},
    "outputs": [],
@@ -270,10 +385,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "id": "bca1c275-cc3d-43df-923e-e6604d584226",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "example = data[\"test\"][0]\n",
     "example"
@@ -281,10 +409,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "id": "76f2e027-0a31-4919-bb7e-404c786e1599",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "prompt = build_prompt(example)\n",
     "print(prompt)"
@@ -292,24 +437,80 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
    "id": "932b54ca-7e27-47cd-b67d-7ef8386b6608",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tokens:  Bel ow Ġ is Ġ an Ġ instruc tion Ġ th at Ġ describ es Ġ a Ġ t ask . ĠWrit e Ġ a Ġ respon se Ġ th at Ġ appropri at el y Ġ complet es Ġ the Ġ request . Ċ Ċ # # # ĠIn struc tion : Ċ D eter min e Ġ cor e Ġ promo ter Ġ det ec tion Ġ of Ġ follow ing Ġ d na Ġ sequenc e , ĠTh e Ġ resul t Ġ will Ġ be Ġ on e Ġ of Ġ the Ġ follow ing : ĠN on - promo ter , Ġ promo ter . Ċ Ċ # # # ĠIn put : Ċ CC AGGATGC GC TGACG ACCC GGCTGGC AGGC GGGTCC TCG TGGGCG AGGCG AGGGAGGC GGCG AGAGAGG AGCAATAG TTTCCC ACCGC TCCCTCTC AGGCGC AGGG TCTAG AGAAGC GCG AGGGG ATCTAG AGAAGCC GG AGGGG AGGAAGC GCG AGTCC GCGG CCCGCC CCG TTGCG TCCC ACCCACC GCG TCCCCTCCCC TCCCCTCCC GCTGC GGG AAAAGC GGCCGC GGGCGGC GGCGCCC ACTGTG GGGC GGGC GGAGC GCCGC GGGAGGC GGACG AGATGCG AGCGC GGCCGC Ċ Ċ # # # ĠR esp on se : Ċ promo ter\n"
+     ]
+    }
+   ],
    "source": [
     "print('tokens: ', ' '.join(tokenizer.tokenize(prompt)))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "id": "26671faf-68d0-4a44-978e-e1a24e86c9b1",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "02d2f083e74a45e6ada46b9872822dfc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/53275 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f9ebfc901049446cb70580ee6d90861b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/5920 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['input_ids', 'attention_mask'],\n",
+       "        num_rows: 53275\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['input_ids', 'attention_mask'],\n",
+       "        num_rows: 5920\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "def tokenize_function(example):\n",
     "    prompt =  build_prompt(example)\n",
-    "    result = tokenizer(prompt, padding='max_length', truncation=True, max_length=1024) # max_length=1024\n",
+    "    result = tokenizer(prompt, padding='max_length', truncation=True, max_length=256) # max_length=256\n",
     "    return result\n",
     "\n",
     "\n",
@@ -322,17 +523,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "id": "3d46c8b1-9fb3-431a-87ea-c278468543e7",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['input_ids', 'attention_mask'],\n",
+       "    num_rows: 53275\n",
+       "})"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenized_datasets[\"train\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "id": "26985c81-4335-4ac0-9a5a-84a5b4f2d0e4",
    "metadata": {},
    "outputs": [],
@@ -346,17 +561,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "id": "e18d3095-d6dd-423b-84fb-dca4a629d450",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "08a24dc5ff954b8581d5c2378e0d60d6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/111 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "model = GPT2LMHeadModel.from_pretrained(\"dnagpt/gene_eng_gpt2_v0\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 22,
    "id": "12134cf2-676a-4176-a733-35caab2fd520",
    "metadata": {},
    "outputs": [],
@@ -401,10 +631,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
    "id": "b9a2e2a9-a1ff-44b0-a550-623a16d0d7a2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n",
+      "TATAT\n"
+     ]
+    }
+   ],
    "source": [
     "input_text = format_input(data[\"test\"][0])\n",
     "\n",
@@ -418,7 +679,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 24,
    "id": "63b54fe2-f077-4ca8-974e-1bcc41ce57d6",
    "metadata": {},
    "outputs": [],
@@ -437,10 +698,217 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 25,
    "id": "61df123d-e67d-4548-998a-de1e2781e774",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-01-10 15:41:26,331] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/compiler_compat/ld: cannot find -laio: No such file or directory\n",
+      "collect2: error: ld returned 1 exit status\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libpthread.so.0, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libstdc++.so.6, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: warning: libm.so.6, needed by /usr/local/cuda/lib64/libcufile.so, not found (try using -rpath or -rpath-link)\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::runtime_error::~runtime_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__gxx_personality_v0@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::tellp()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::chrono::_V2::steady_clock::now()@GLIBCXX_3.4.19'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_replace_aux(unsigned long, unsigned long, unsigned long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for bool@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::logic_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::~locale()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_end_catch@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ofstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::logic_error::~logic_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for __cxxabiv1::__si_class_type_info@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::_M_cache_locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator new[](unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_leak_hard()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ifstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >::basic_streambuf(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned short@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::resize(unsigned long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for char const*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ctype<char>::_M_widen_init() const@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_invalid_argument(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::operator=(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<wchar_t, std::char_traits<wchar_t> >::_M_cache_locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_decrement(std::_Rb_tree_node_base const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_free_exception@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::notify_one()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::Init::~Init()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_pure_virtual@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::flush()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for __cxxabiv1::__class_type_info@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_rethrow@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_fstream<char, std::char_traits<char> >::~basic_fstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::compare(char const*) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ostringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::locale()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::chrono::_V2::system_clock::now()@GLIBCXX_3.4.19'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_ifstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Hash_bytes(void const*, unsigned long, unsigned long)@CXXABI_1.3.5'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<long long>(long long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for char*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_Prime_rehash_policy::_M_need_rehash(unsigned long, unsigned long, unsigned long) const@GLIBCXX_3.4.18'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::out_of_range@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<unsigned long>(unsigned long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_increment(std::_Rb_tree_node_base const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::~ios_base()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::range_error::~range_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__basic_file<char>::~__basic_file()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_guard_acquire@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<bool>(bool)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::overflow_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_fstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::range_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ios<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_filebuf<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator delete[](void*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(unsigned long, char, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_transfer(std::__detail::_List_node_base*, std::__detail::_List_node_base*)@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::replace(unsigned long, unsigned long, char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for std::exception@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_Rep::_M_destroy(std::allocator<wchar_t> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream& std::istream::_M_extract<double>(double&)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_fstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::basic_ifstream(char const*, std::_Ios_Openmode)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator new(unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_istringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned int@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::append(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::domain_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::find(char, unsigned long) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::put(char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for int@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_bad_alloc()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_thread_atexit@CXXABI_1.3.7'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_increment(std::_Rb_tree_node_base*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::~basic_ifstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::Init::Init()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::condition_variable()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::basic_filebuf()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::domain_error::~domain_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::cerr@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::find(char const*, unsigned long, unsigned long) const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::str() const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::invalid_argument@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for void*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::assign(std::string const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_ostringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_rebalance_for_erase(std::_Rb_tree_node_base*, std::_Rb_tree_node_base&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_hook(std::__detail::_List_node_base*)@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__detail::_List_node_base::_M_unhook()@GLIBCXX_3.4.15'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ostringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::_M_sync(char*, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_iostream<char, std::char_traits<char> >::~basic_iostream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::locale::locale(std::locale const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_istringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `log2f@GLIBC_2.2.5'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::operator<<(std::basic_streambuf<char, std::char_traits<char> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::exception::~exception()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__basic_file<char>::is_open() const@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_istringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_istringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::swap(std::string&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<char, std::char_traits<char> >::basic_streambuf(std::basic_streambuf<char, std::char_traits<char> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::init(std::basic_streambuf<char, std::char_traits<char> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_bad_cast()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<char, std::char_traits<char> >::clear(std::_Ios_Iostate)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >::operator=(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `operator delete(void*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream::operator<<(int)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_S_empty_rep_storage@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_Rep::_M_destroy(std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_iostream<wchar_t, std::char_traits<wchar_t> >::~basic_iostream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::runtime_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ofstream<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_insert_and_rebalance(bool, std::_Rb_tree_node_base*, std::_Rb_tree_node_base*, std::_Rb_tree_node_base&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_stringstream()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `VTT for std::basic_stringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<long>(long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream::get()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned long long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::operator<< <std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::out_of_range::~out_of_range()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::length_error::~length_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::invalid_argument::~invalid_argument()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::swap(std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::cout@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<unsigned long long>(unsigned long long)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for int*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<void const*>(void const*)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::underflow_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_streambuf<char, std::char_traits<char> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for std::out_of_range@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_allocate_exception@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_ios<wchar_t, std::char_traits<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for void const*@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ios<wchar_t, std::char_traits<wchar_t> >::init(std::basic_streambuf<wchar_t, std::char_traits<wchar_t> >*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::reserve(unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_begin_catch@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_Rep::_S_empty_rep_storage@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::_M_leak()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::open(char const*, std::_Ios_Openmode)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_stringbuf<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::_M_sync(wchar_t*, unsigned long, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::istream::getline(char*, long, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_istream<char, std::char_traits<char> >& std::getline<char, std::char_traits<char>, std::allocator<char> >(std::basic_istream<char, std::char_traits<char> >&, std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, char)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::condition_variable::~condition_variable()@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::basic_stringbuf<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::insert(unsigned long, char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::string::assign(char const*, unsigned long)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for unsigned char@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ios_base::ios_base()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_out_of_range(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::overflow_error::~overflow_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_length_error(char const*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::__throw_system_error(int)@GLIBCXX_3.4.11'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ofstream<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::ostream& std::ostream::_M_insert<double>(double)@GLIBCXX_3.4.9'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_streambuf<char, std::char_traits<char> >::operator=(std::basic_streambuf<char, std::char_traits<char> > const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `typeinfo for long long@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, unsigned long, std::allocator<char> const&)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_ifstream<char, std::char_traits<char> >::close()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_guard_release@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `__cxa_throw@CXXABI_1.3'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::underflow_error::~underflow_error()@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::_Rb_tree_decrement(std::_Rb_tree_node_base*)@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `vtable for std::length_error@GLIBCXX_3.4'\n",
+      "/root/miniconda3/compiler_compat/ld: /usr/local/cuda/lib64/libcufile.so: undefined reference to `std::basic_filebuf<char, std::char_traits<char> >::~basic_filebuf()@GLIBCXX_3.4'\n",
+      "collect2: error: ld returned 1 exit status\n"
+     ]
+    }
+   ],
    "source": [
     "# 初始化Trainer\n",
     "trainer = Trainer(\n",
@@ -454,10 +922,204 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 26,
    "id": "a9cd936a-5ea6-43e3-9848-27080f818606",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='19980' max='19980' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [19980/19980 21:55, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>2.335700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1000</td>\n",
+       "      <td>2.184100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1500</td>\n",
+       "      <td>2.178500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2000</td>\n",
+       "      <td>2.172400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2500</td>\n",
+       "      <td>2.171400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3000</td>\n",
+       "      <td>2.171900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3500</td>\n",
+       "      <td>2.163100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4000</td>\n",
+       "      <td>2.159300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4500</td>\n",
+       "      <td>2.161000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5000</td>\n",
+       "      <td>2.160200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5500</td>\n",
+       "      <td>2.158400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6000</td>\n",
+       "      <td>2.151600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6500</td>\n",
+       "      <td>2.153700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7000</td>\n",
+       "      <td>2.129500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7500</td>\n",
+       "      <td>2.119100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8000</td>\n",
+       "      <td>2.119800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8500</td>\n",
+       "      <td>2.121600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9000</td>\n",
+       "      <td>2.122500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9500</td>\n",
+       "      <td>2.122300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10000</td>\n",
+       "      <td>2.121500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10500</td>\n",
+       "      <td>2.119500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11000</td>\n",
+       "      <td>2.123500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11500</td>\n",
+       "      <td>2.119600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12000</td>\n",
+       "      <td>2.119300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12500</td>\n",
+       "      <td>2.121800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13000</td>\n",
+       "      <td>2.123500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13500</td>\n",
+       "      <td>2.103200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14000</td>\n",
+       "      <td>2.080700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14500</td>\n",
+       "      <td>2.082100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15000</td>\n",
+       "      <td>2.082900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15500</td>\n",
+       "      <td>2.086400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16000</td>\n",
+       "      <td>2.086600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16500</td>\n",
+       "      <td>2.083800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17000</td>\n",
+       "      <td>2.085000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17500</td>\n",
+       "      <td>2.082800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18000</td>\n",
+       "      <td>2.077600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18500</td>\n",
+       "      <td>2.080300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19000</td>\n",
+       "      <td>2.086600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19500</td>\n",
+       "      <td>2.084200</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=19980, training_loss=2.1272921145021977, metrics={'train_runtime': 1315.5944, 'train_samples_per_second': 121.485, 'train_steps_per_second': 15.187, 'total_flos': 2.08804995072e+16, 'train_loss': 2.1272921145021977, 'epoch': 3.0})"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "# 开始训练\n",
     "trainer.train()"
@@ -465,10 +1127,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 27,
    "id": "315aae76-44b4-4513-8139-40ef22934873",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved model to: gpt_ft/final\n"
+     ]
+    }
+   ],
    "source": [
     "save_dir = 'gpt_ft/final'\n",
     "trainer.save_model(save_dir)\n",
@@ -477,7 +1147,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 28,
    "id": "28d2dbbc-02ff-4120-b230-b19905a786cd",
    "metadata": {},
    "outputs": [],
@@ -488,20 +1158,92 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 29,
    "id": "08987c3c-063a-4e9b-9ebb-e637b0b5bccd",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(90000, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=90000, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "finetuned_model"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 30,
    "id": "d75010e8-6d6a-40ef-852e-0d705adc3da8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n",
+      "promoterpromoterpromo\n",
+      "--------------------------\n",
+      "\n",
+      "real answer: \n",
+      "\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "print(\"input (test):\", input_text)\n",
     "\n",
@@ -517,10 +1259,217 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 31,
    "id": "64365e15-510e-4abf-92f5-c78b660b37dc",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n"
+     ]
+    }
+   ],
    "source": [
     "test_data = data[\"test\"].select(range(100))\n",
     "\n",
@@ -543,7 +1492,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 32,
    "id": "a45fb780-fc3f-401c-b6e0-6f7d0c1682de",
    "metadata": {},
    "outputs": [],
@@ -556,19 +1505,128 @@
     "# 将 Dataset 对象导出为 JSON 文件\n",
     "# test_data.to_json(output_file)\n",
     "with open(output_file, \"w\") as file:\n",
-    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing"
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "a83c8881-c763-4bba-8b85-584a6722a38e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "promoter |||||||||||| promoterpromoterpromo\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "presicion 0.94 same 0.49\n"
+     ]
+    }
+   ],
    "source": [
     "import json\n",
     "\n",
     "\n",
+    "output_file = 'gpt2-small3-1024.json'\n",
     "\n",
     "with open(output_file, \"r\") as file:\n",
     "    test_data = json.load(file)\n",
@@ -580,15 +1638,30 @@
     "    output = item[\"output\"]\n",
     "    #output = \" \".join(tokenizer.tokenize(output))\n",
     "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
     "    if model_response == output: #same it\n",
     "        same_sum = same_sum + 1\n",
     "        \n",
-    "    if model_response.find(output)!=-1: #find it\n",
-    "        right_sum = right_sum + 1\n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
     "\n",
     "\n",
-    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)"
+    "print(\"Accuracy\", right_sum/all_num, \"same\", same_sum/all_num)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8bf88885-fb33-406e-9644-cd174a8a2f28",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/4-deepspeed-intro.ipynb

@@ -10,12 +10,172 @@
   },
   {
    "cell_type": "markdown",
-   "id": "3383c2d7-91a9-4940-b3b2-698fb7d9dbb7",
+   "id": "75b8219d-8069-4b18-96c8-d5024ee049f1",
    "metadata": {},
    "source": [
-    "## 使用gpt2+deepspeed训练"
+    "## 大模型并行训练简介\n",
+    "\n",
+    "大模型的并行训练旨在克服单个 GPU 显存的限制和加速训练过程，通常适用于参数规模较大的模型（如 GPT-3、T5 等）。并行训练主要包括以下几种方法，每种方法适用于不同的场景和模型特性。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 数据并行（Data Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将数据切分成多个小批次，每个 GPU 处理其中一部分。\n",
+    "- 模型副本被复制到每个 GPU。\n",
+    "- 每个 GPU 独立计算梯度，最终通过梯度同步（如 AllReduce 操作）更新参数。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 实现简单，是最常用的并行方法。\n",
+    "  - 对模型大小没有限制。\n",
+    "- **缺点**：\n",
+    "  - 模型副本需要完整加载到每个 GPU，占用显存。\n",
+    "  - 在超大规模模型中，显存压力较大。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 参数规模适中，显存可以容纳整个模型的场景。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 模型并行（Model Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型切分成不同的部分，将不同部分分配到不同的 GPU。\n",
+    "- 前向传播和后向传播时，数据在模型的不同部分之间传递。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 不需要复制整个模型，可以支持超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - GPU 之间通信频繁，可能成为性能瓶颈。\n",
+    "  - 实现复杂，切分模型需要精心设计。\n",
+    "  \n",
+    "#### **适用场景**\n",
+    "- 单个 GPU 无法容纳完整模型参数的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- 将 Transformer 的不同层分配到不同的 GPU。\n",
+    "- 常用工具：DeepSpeed 的 Pipeline Parallelism、NVIDIA Megatron-LM。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 张量并行（Tensor Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型内部的张量（如权重矩阵）切分为多个子张量，并分配到不同 GPU。\n",
+    "- GPU 之间协作完成矩阵计算。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 减少了每个 GPU 的显存占用，同时保持模型整体完整性。\n",
+    "- **缺点**：\n",
+    "  - 实现较复杂，需要优化通信操作。\n",
+    "  - 通信开销较高，适合较大批量的训练。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 参数非常大的模型（如 GPT-3）。\n",
+    "- 需要极致优化显存的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- NVIDIA 的 Megatron-LM 和 Hugging Face Transformers 提供了张量并行的支持。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 管道并行（Pipeline Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将模型分为不同的部分（通常是按层划分），每部分分配到不同的 GPU。\n",
+    "- 数据按照流水线的方式流经每个 GPU。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 减少每个 GPU 的显存压力。\n",
+    "  - 通过流水线增加计算效率。\n",
+    "- **缺点**：\n",
+    "  - 引入流水线延迟。\n",
+    "  - 实现复杂，需管理数据依赖和同步。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 模型非常深，层数较多的场景。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- DeepSpeed 的 Pipeline Parallelism。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 混合并行（Hybrid Parallelism）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 将数据并行、模型并行、张量并行和管道并行组合使用，充分利用多 GPU 资源。\n",
+    "- 不同的并行方法在不同维度协同工作。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 灵活且适应性强，适合超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - 配置复杂，依赖于框架和训练任务。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 超大规模模型（如 GPT-3 或参数量 >1T）。\n",
+    "- 多机多卡的大型训练环境。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- NVIDIA Megatron-LM 和 DeepSpeed 的混合并行支持。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. ZeRO 优化并行（Zero Redundancy Optimizer）**\n",
+    "\n",
+    "#### **原理**\n",
+    "- 通过分片存储模型参数、优化器状态和梯度，显著减少每个 GPU 的显存占用。\n",
+    "\n",
+    "#### **特点**\n",
+    "- **优点**：\n",
+    "  - 极大降低显存需求。\n",
+    "  - 支持超大规模模型。\n",
+    "- **缺点**：\n",
+    "  - 对 GPU 间通信要求较高。\n",
+    "  - 比数据并行复杂。\n",
+    "\n",
+    "#### **适用场景**\n",
+    "- 超大模型的高效训练。\n",
+    "\n",
+    "#### **具体实现**\n",
+    "- DeepSpeed 提供的 ZeRO Stage 1/2/3。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **方法对比**\n",
+    "\n",
+    "| 并行方法      | 主要优点                      | 主要缺点                      | 适用场景                   |\n",
+    "|---------------|-------------------------------|-------------------------------|---------------------------|\n",
+    "| 数据并行      | 简单高效，易实现              | 模型副本占用大量显存          | 模型规模适中，显存足够     |\n",
+    "| 模型并行      | 支持大模型                    | 通信开销大，切分复杂          | 超大模型，显存有限         |\n",
+    "| 张量并行      | 高效利用显存                  | 实现复杂，通信频繁            | 参数规模极大的模型         |\n",
+    "| 管道并行      | 显存需求降低，适合深模型      | 流水线延迟，数据同步复杂      | 层数多的大型模型           |\n",
+    "| 混合并行      | 灵活适配超大规模模型          | 配置复杂，依赖框架            | 超大规模模型（如 GPT-3）   |\n",
+    "| ZeRO 并行     | 极大节省显存，占用少          | 通信成本高                    | 超大规模模型显存优化       |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "- **中等规模模型**：优先使用 **数据并行**。\n",
+    "- **单卡显存不足**：采用 **模型并行** 或 **张量并行**。\n",
+    "- **超大规模模型**：使用 **混合并行** 或 DeepSpeed 的 **ZeRO 优化**。\n",
+    "\n",
+    "对于现代超大规模模型，通常采用混合并行方法，比如 NVIDIA 的 Megatron-LM 和微软的 DeepSpeed，它们综合了多种并行策略，能够有效利用计算资源并加速训练。如果您有具体的硬件环境或模型需求，可以进一步探讨适合的并行方案！"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06ddaa4d-e04a-41e0-beb5-f04dfaebcd54",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "id": "c0d29667-1e75-46df-8f65-cae27609ee3f",
@@ -169,171 +329,19 @@
    ]
   },
   {
-   "cell_type": "markdown",
-   "id": "75b8219d-8069-4b18-96c8-d5024ee049f1",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5372798-ced3-420c-b853-badd3ff05dc1",
    "metadata": {},
-   "source": [
-    "## 大模型并行方法\n",
-    "\n",
-    "大模型的并行训练旨在克服单个 GPU 显存的限制和加速训练过程，通常适用于参数规模较大的模型（如 GPT-3、T5 等）。并行训练主要包括以下几种方法，每种方法适用于不同的场景和模型特性。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **1. 数据并行（Data Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将数据切分成多个小批次，每个 GPU 处理其中一部分。\n",
-    "- 模型副本被复制到每个 GPU。\n",
-    "- 每个 GPU 独立计算梯度，最终通过梯度同步（如 AllReduce 操作）更新参数。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 实现简单，是最常用的并行方法。\n",
-    "  - 对模型大小没有限制。\n",
-    "- **缺点**：\n",
-    "  - 模型副本需要完整加载到每个 GPU，占用显存。\n",
-    "  - 在超大规模模型中，显存压力较大。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 参数规模适中，显存可以容纳整个模型的场景。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **2. 模型并行（Model Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型切分成不同的部分，将不同部分分配到不同的 GPU。\n",
-    "- 前向传播和后向传播时，数据在模型的不同部分之间传递。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 不需要复制整个模型，可以支持超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - GPU 之间通信频繁，可能成为性能瓶颈。\n",
-    "  - 实现复杂，切分模型需要精心设计。\n",
-    "  \n",
-    "#### **适用场景**\n",
-    "- 单个 GPU 无法容纳完整模型参数的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- 将 Transformer 的不同层分配到不同的 GPU。\n",
-    "- 常用工具：DeepSpeed 的 Pipeline Parallelism、NVIDIA Megatron-LM。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **3. 张量并行（Tensor Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型内部的张量（如权重矩阵）切分为多个子张量，并分配到不同 GPU。\n",
-    "- GPU 之间协作完成矩阵计算。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 减少了每个 GPU 的显存占用，同时保持模型整体完整性。\n",
-    "- **缺点**：\n",
-    "  - 实现较复杂，需要优化通信操作。\n",
-    "  - 通信开销较高，适合较大批量的训练。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 参数非常大的模型（如 GPT-3）。\n",
-    "- 需要极致优化显存的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- NVIDIA 的 Megatron-LM 和 Hugging Face Transformers 提供了张量并行的支持。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **4. 管道并行（Pipeline Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将模型分为不同的部分（通常是按层划分），每部分分配到不同的 GPU。\n",
-    "- 数据按照流水线的方式流经每个 GPU。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 减少每个 GPU 的显存压力。\n",
-    "  - 通过流水线增加计算效率。\n",
-    "- **缺点**：\n",
-    "  - 引入流水线延迟。\n",
-    "  - 实现复杂，需管理数据依赖和同步。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 模型非常深，层数较多的场景。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- DeepSpeed 的 Pipeline Parallelism。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **5. 混合并行（Hybrid Parallelism）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 将数据并行、模型并行、张量并行和管道并行组合使用，充分利用多 GPU 资源。\n",
-    "- 不同的并行方法在不同维度协同工作。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 灵活且适应性强，适合超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - 配置复杂，依赖于框架和训练任务。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 超大规模模型（如 GPT-3 或参数量 >1T）。\n",
-    "- 多机多卡的大型训练环境。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- NVIDIA Megatron-LM 和 DeepSpeed 的混合并行支持。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **6. ZeRO 优化并行（Zero Redundancy Optimizer）**\n",
-    "\n",
-    "#### **原理**\n",
-    "- 通过分片存储模型参数、优化器状态和梯度，显著减少每个 GPU 的显存占用。\n",
-    "\n",
-    "#### **特点**\n",
-    "- **优点**：\n",
-    "  - 极大降低显存需求。\n",
-    "  - 支持超大规模模型。\n",
-    "- **缺点**：\n",
-    "  - 对 GPU 间通信要求较高。\n",
-    "  - 比数据并行复杂。\n",
-    "\n",
-    "#### **适用场景**\n",
-    "- 超大模型的高效训练。\n",
-    "\n",
-    "#### **具体实现**\n",
-    "- DeepSpeed 提供的 ZeRO Stage 1/2/3。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **方法对比**\n",
-    "\n",
-    "| 并行方法      | 主要优点                      | 主要缺点                      | 适用场景                   |\n",
-    "|---------------|-------------------------------|-------------------------------|---------------------------|\n",
-    "| 数据并行      | 简单高效，易实现              | 模型副本占用大量显存          | 模型规模适中，显存足够     |\n",
-    "| 模型并行      | 支持大模型                    | 通信开销大，切分复杂          | 超大模型，显存有限         |\n",
-    "| 张量并行      | 高效利用显存                  | 实现复杂，通信频繁            | 参数规模极大的模型         |\n",
-    "| 管道并行      | 显存需求降低，适合深模型      | 流水线延迟，数据同步复杂      | 层数多的大型模型           |\n",
-    "| 混合并行      | 灵活适配超大规模模型          | 配置复杂，依赖框架            | 超大规模模型（如 GPT-3）   |\n",
-    "| ZeRO 并行     | 极大节省显存，占用少          | 通信成本高                    | 超大规模模型显存优化       |\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **总结**\n",
-    "- **中等规模模型**：优先使用 **数据并行**。\n",
-    "- **单卡显存不足**：采用 **模型并行** 或 **张量并行**。\n",
-    "- **超大规模模型**：使用 **混合并行** 或 DeepSpeed 的 **ZeRO 优化**。\n",
-    "\n",
-    "对于现代超大规模模型，通常采用混合并行方法，比如 NVIDIA 的 Megatron-LM 和微软的 DeepSpeed，它们综合了多种并行策略，能够有效利用计算资源并加速训练。如果您有具体的硬件环境或模型需求，可以进一步探讨适合的并行方案！"
-   ]
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "markdown",
    "id": "cd848439-bac8-46b2-9a0f-59ae7c343954",
    "metadata": {},
    "source": [
-    "## deepspeed的并行设计\n",
+    "## deepspeed具体设置\n",
     "\n",
     "\n",
     "是的，DeepSpeed 支持多种并行策略，包括 **数据并行**、**模型并行** 和 **张量并行**，并且可以通过其��置文件灵活地设置这些并行模式。\n",
@@ -544,6 +552,22 @@
     "DeepSpeed 的配置高度灵活，可以根据模型大小、显存限制和硬件条件选择适合的并行策略。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a8e6de4c-adc1-4a1b-840a-c8542b4ed783",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3383c2d7-91a9-4940-b3b2-698fb7d9dbb7",
+   "metadata": {},
+   "source": [
+    "## 使用gpt2+deepspeed训练"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "ab2812bc-f743-4f18-b49c-972781484dc6",
@@ -569,14 +593,6 @@
    "metadata": {},
    "outputs": [],
    "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ce701aeb-c8c7-450a-bbf9-b793a19cd0c6",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

04-gene-sft/5-peft-intro.ipynb

@@ -8,6 +8,14 @@
     "# 4.5 peft简介"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "f4288594-c676-4369-aca1-730446f293d7",
+   "metadata": {},
+   "source": [
+    "## peft"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "182b82c4-d484-4c15-a600-03c3b51367ec",
@@ -139,6 +147,22 @@
     "如果您需要实现高效微调，可以结合 Hugging Face 的 PEFT 库快速上手。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a70b2631-c9b9-49da-96c6-6760c63040ac",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b47ddf3-85c9-4dd8-bbbb-34fc3bd6aa1b",
+   "metadata": {},
+   "source": [
+    "## GPT2使用peft样例"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -366,14 +390,6 @@
     "    print(name)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "37aa6abb-ab1c-4e9c-b968-579dd74044db",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "id": "0add2f79-f35c-4638-80bb-0d8a87a9b6a7",
@@ -502,11 +518,19 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b4a41750-420f-49c4-845d-69db394794f9",
+   "id": "14f20171-0719-4dfa-b888-147b657ebff4",
    "metadata": {},
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "markdown",
+   "id": "b4e7bff2-2a4f-4a1d-9cb1-dd02aead2f85",
+   "metadata": {},
+   "source": [
+    "## LoraConfig具体配置"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "10c99eb9-8007-4297-972e-7be71768c9c3",
@@ -595,62 +619,6 @@
     "\n",
     "---\n",
     "\n",
-    "### **2. 微调哪些参数，冻结哪些参数**\n",
-    "\n",
-    "LoRA 的核心思想是通过 **分解矩阵**，只更新少量参数，而冻结模型的大部分参数。以下是常见设置的说明：\n",
-    "\n",
-    "#### **微调的参数**\n",
-    "- LoRA 通过 `target_modules` 指定的模块，例如：\n",
-    "  - GPT-2 的 `c_attn`（自注意力模块）。\n",
-    "  - BERT 的 `query` 和 `key`。\n",
-    "- 这些模块是模型中对性能贡献最大的部分，通过微调这些模块，任务性能可以显著提升。\n",
-    "\n",
-    "#### **冻结的参数**\n",
-    "- 除了 `target_modules` 中指定的参数外，所有其他模型参数默认冻结，包括：\n",
-    "  - 预训练权重的绝大部分。\n",
-    "  - 偏置参数（如果 `bias=\"none\"`）。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **3. 一般如何设置**\n",
-    "\n",
-    "#### **（1）针对不同任务调整**\n",
-    "- **文本分类任务**：\n",
-    "  - 优先选择自注意力模块（如 `c_attn`）作为 `target_modules`。\n",
-    "  - `r=8` 或 `r=16` 是常见选择，适中计算开销。\n",
-    "  - 设置适当的 dropout（如 `lora_dropout=0.1`）以防止过拟合。\n",
-    "  \n",
-    "- **语言生成任务**：\n",
-    "  - 对 GPT-2 或 GPT-3，选择 `q_proj` 和 `v_proj`（query 和 value 投影模块）。\n",
-    "  - `r=16` 或更高，适应生成任务的高复杂性。\n",
-    "\n",
-    "- **命名实体识别任务**：\n",
-    "  - 优先选择 `q_proj` 和 `k_proj`（query 和 key 模块）。\n",
-    "\n",
-    "#### **（2）参数量与显存的权衡**\n",
-    "- 如果显存有限，减少 `r` 的值。\n",
-    "- 对小型任务，`r=4` 或 `r=8` 通常已经足够。\n",
-    "\n",
-    "#### **（3）偏置设置**\n",
-    "- 偏置参数的影响较小，在大多数情况下，可以选择 `bias=\"none\"` 保持冻结。\n",
-    "- 对非常依赖偏置的任务（如生成风格微调），可以尝试 `bias=\"lora_only\"`。\n",
-    "\n",
-    "---\n",
-    "\n",
-    "### **4. 示例：如何选择目标模块**\n",
-    "\n",
-    "#### **GPT-2**\n",
-    "对 GPT-2 来说，以下模块通常是微调的目标：\n",
-    "- **`c_attn`**：注意力模块的组合层。\n",
-    "- **`q_proj` 和 `v_proj`**：Query 和 Value 的线性投影。\n",
-    "\n",
-    "#### **BERT**\n",
-    "对 BERT 来说，以下模块通常是微调的目标：\n",
-    "- **`query`**：Attention 的 Query 模块。\n",
-    "- **`key`**：Attention 的 Key 模块。\n",
-    "\n",
-    "---\n",
-    "\n",
     "### **5. 总结建议**\n",
     "- **微调的参数**：优先选择模型中注意力相关模块。\n",
     "- **冻结的参数**：大部分参数默认冻结以节省显存。\n",
@@ -664,22 +632,82 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "execution_count": 1,
+   "id": "bbc080ba-3ee8-4bc6-afd9-2a3241f1bcda",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at dnagpt/dna_gpt2_v0 and are newly initialized: ['score.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f7e72521368341d38a2b11028715a871",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/5920 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "trainable params: 296,448 || all params: 109,180,416 || trainable%: 0.2715\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/peft/tuners/lora/layer.py:1264: UserWarning: fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
    "source": [
     "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer\n",
     "from peft import LoraConfig, get_peft_model, TaskType\n",
     "from datasets import load_dataset\n",
     "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
+    "from transformers import DataCollatorWithPadding\n",
     "\n",
     "# **1. 加载模型和分词器**\n",
-    "model_name = \"gpt2\"  # 基础模型\n",
+    "model_name = \"dnagpt/dna_gpt2_v0\"  # 基础模型\n",
     "num_labels = 2       # 二分类任务\n",
     "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)\n",
     "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
-    "tokenizer.pad_token = tokenizer.eos_token  # 设置 pad_token 为 eos_token\n",
+    "\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "model.config.pad_token_id = tokenizer.pad_token_id\n",
+    "\n",
     "\n",
     "# **2. 定义数据集**\n",
     "# 示例数据集：dna_promoter_300\n",
@@ -693,7 +721,10 @@
     "    )\n",
     "\n",
     "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
-    "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "#tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "\n",
+    "# 4. 创建一个数据收集器，用于动态填充和遮蔽\n",
+    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
     "\n",
     "# **4. 划分数据集**\n",
     "train_dataset = tokenized_datasets[\"train\"]\n",
@@ -711,8 +742,156 @@
     "\n",
     "# 使用 LoRA 包装模型\n",
     "model = get_peft_model(model, lora_config)\n",
-    "model.print_trainable_parameters()  # 打印可训练的参数信息\n",
-    "\n",
+    "model.print_trainable_parameters()  # 打印可训练的参数信息"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "7da39e7f-db92-483c-888d-19707ab35c5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
+      "  warnings.warn(\n",
+      "/tmp/ipykernel_2399/3695291394.py:28: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='66600' max='66600' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [66600/66600 34:07, Epoch 10/10]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>Precision</th>\n",
+       "      <th>Recall</th>\n",
+       "      <th>F1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>0.268300</td>\n",
+       "      <td>0.307843</td>\n",
+       "      <td>0.909797</td>\n",
+       "      <td>0.916809</td>\n",
+       "      <td>0.901987</td>\n",
+       "      <td>0.909338</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>0.287400</td>\n",
+       "      <td>0.278804</td>\n",
+       "      <td>0.913514</td>\n",
+       "      <td>0.901339</td>\n",
+       "      <td>0.929269</td>\n",
+       "      <td>0.915091</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>0.282800</td>\n",
+       "      <td>0.291222</td>\n",
+       "      <td>0.914527</td>\n",
+       "      <td>0.913116</td>\n",
+       "      <td>0.916807</td>\n",
+       "      <td>0.914958</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>0.255200</td>\n",
+       "      <td>0.281572</td>\n",
+       "      <td>0.916385</td>\n",
+       "      <td>0.896474</td>\n",
+       "      <td>0.942068</td>\n",
+       "      <td>0.918706</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>0.252000</td>\n",
+       "      <td>0.271950</td>\n",
+       "      <td>0.914527</td>\n",
+       "      <td>0.913116</td>\n",
+       "      <td>0.916807</td>\n",
+       "      <td>0.914958</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>0.242300</td>\n",
+       "      <td>0.288199</td>\n",
+       "      <td>0.916385</td>\n",
+       "      <td>0.916498</td>\n",
+       "      <td>0.916807</td>\n",
+       "      <td>0.916653</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>0.253500</td>\n",
+       "      <td>0.268673</td>\n",
+       "      <td>0.918750</td>\n",
+       "      <td>0.909480</td>\n",
+       "      <td>0.930616</td>\n",
+       "      <td>0.919927</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>0.235900</td>\n",
+       "      <td>0.277893</td>\n",
+       "      <td>0.917568</td>\n",
+       "      <td>0.906855</td>\n",
+       "      <td>0.931290</td>\n",
+       "      <td>0.918910</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>0.238600</td>\n",
+       "      <td>0.280647</td>\n",
+       "      <td>0.917568</td>\n",
+       "      <td>0.913362</td>\n",
+       "      <td>0.923206</td>\n",
+       "      <td>0.918258</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>0.237900</td>\n",
+       "      <td>0.284149</td>\n",
+       "      <td>0.917736</td>\n",
+       "      <td>0.913391</td>\n",
+       "      <td>0.923543</td>\n",
+       "      <td>0.918439</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "训练完成，模型已保存至 ./gpt2_lora_text_classification\n"
+     ]
+    }
+   ],
+   "source": [
     "# **6. 计算指标**\n",
     "def compute_metrics(eval_pred):\n",
     "    predictions, labels = eval_pred\n",
@@ -729,7 +908,7 @@
     "    learning_rate=2e-5,                          # 学习率\n",
     "    per_device_train_batch_size=8,               # 每设备的批量大小\n",
     "    per_device_eval_batch_size=8,                # 每设备评估的批量大小\n",
-    "    num_train_epochs=3,                          # 训练轮数\n",
+    "    num_train_epochs=10,                          # 训练轮数\n",
     "    weight_decay=0.01,                           # 权重衰减\n",
     "    logging_dir=\"./logs\",                        # 日志路径\n",
     "    fp16=True,                                   # 启用混合精度训练\n",
@@ -746,6 +925,7 @@
     "    train_dataset=train_dataset,\n",
     "    eval_dataset=test_dataset,\n",
     "    tokenizer=tokenizer,\n",
+    "    data_collator=data_collator,\n",
     "    compute_metrics=compute_metrics,\n",
     ")\n",
     "\n",

04-gene-sft/6-llama-continue-train.ipynb

@@ -330,6 +330,38 @@
     "本节任务是基于llama。训练一个能够处理dna和protein蛋白质数据的基础预训练大模型，数据为第一章中的预训练数据，包括英文数据。"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "aec90d65-ac62-4394-a526-ca62d8bdbad4",
+   "metadata": {},
+   "source": [
+    "## 环境设置\n",
+    "并行环境对transformer、peft等的版本要求比较高，如果版本不匹配可能会出现各种异常问题\n",
+    "之前的课程，都是单GPU运行，一般不存在版本问题，默认安装的都是最新版本。但运行并行环境时，需要确认下版本再运行，本课程运行并行环境如下：\n",
+    "\n",
+    "* Python 3.12.3\n",
+    "* transformers                   4.45.2\n",
+    "* peft                           0.3.0.dev0\n",
+    "* deepspeed                      0.15.2\n",
+    "* accelerate                     1.0.0\n",
+    "\n",
+    "如果不是，可以重新安装即可：\n",
+    "```\n",
+    "pip install transformers==4.45.2 deepspeed==0.15.2 accelerate==1.0.0\n",
+    "\n",
+    "#peft参考使用的是chinese llama的版本，需要git安装\n",
+    "\n",
+    "git clone https://github.com/huggingface/peft.git\n",
+    "\n",
+    "cd peft\n",
+    "\n",
+    "git checkout 13e53fc\n",
+    "\n",
+    "pip install . \n",
+    "```\n",
+    "如果有环境问题，可以查看本目录下的pip_list.txt"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "b1bd33b8-2e05-4b59-9d8f-c48de194cfd6",
@@ -339,8 +371,18 @@
     "\n",
     "```\n",
     "# 复制第一章训练数据,包括dna，protein，还有英文数据，添加英文数据是为了避免遗忘问题\n",
+    "\n",
     "mkdir train_data\n",
     "cp ../01-data_env/data/*.txt train_data/\n",
+    "使用这些数据，6卡4090大概大致需要训练16个小时，autodl也需要近200块钱了。\n",
+    "\n",
+    "建议学习时，可以使用1/10的数据训练：\n",
+    "awk ‘NR%10==1’ dna_1g.txt > dna.txt\n",
+    "rm dna_1g.txt\n",
+    "其他2类数据依次类推\n",
+    "\n",
+    "这样大概需要2到3个小时就能训练完成了\n",
+    "\n",
     "\n",
     "#持续预训练\n",
     "./run_pt.sh\n",
@@ -361,7 +403,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "69b3e97f-a801-4264-a651-a854bcfba9c6",
    "metadata": {},
    "outputs": [],
@@ -376,10 +418,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "339435d9-9379-4b30-ae8b-50feee1ba714",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LlamaTokenizer(name_or_path='dnahlm-merge-hf', vocab_size=91643, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '</s>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
+       "\t0: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "\t1: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
+       "}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-merge-hf\")\n",
     "tokenizer.pad_token = tokenizer.eos_token\n",
@@ -388,10 +445,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "d0f154bb-b1ab-4611-a14c-9b403043fd96",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "342e4ab139b64bb78f0429c2f92c8310",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(91643, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=91643, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "model = LlamaForCausalLM.from_pretrained(\"dnahlm-merge-hf\") #continue pretrain\n",
     "model"
@@ -399,10 +507,51 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "792a9f78-1828-4695-9f6e-479a704ea7e8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LlamaConfig {\n",
+       "  \"_name_or_path\": \"dnahlm-merge-hf\",\n",
+       "  \"architectures\": [\n",
+       "    \"LlamaForCausalLM\"\n",
+       "  ],\n",
+       "  \"attention_bias\": false,\n",
+       "  \"attention_dropout\": 0.0,\n",
+       "  \"bos_token_id\": 1,\n",
+       "  \"eos_token_id\": 2,\n",
+       "  \"head_dim\": 128,\n",
+       "  \"hidden_act\": \"silu\",\n",
+       "  \"hidden_size\": 4096,\n",
+       "  \"initializer_range\": 0.02,\n",
+       "  \"intermediate_size\": 11008,\n",
+       "  \"max_position_embeddings\": 2048,\n",
+       "  \"mlp_bias\": false,\n",
+       "  \"model_type\": \"llama\",\n",
+       "  \"num_attention_heads\": 32,\n",
+       "  \"num_hidden_layers\": 32,\n",
+       "  \"num_key_value_heads\": 32,\n",
+       "  \"pad_token_id\": 0,\n",
+       "  \"pretraining_tp\": 1,\n",
+       "  \"rms_norm_eps\": 1e-06,\n",
+       "  \"rope_scaling\": null,\n",
+       "  \"rope_theta\": 10000.0,\n",
+       "  \"tie_word_embeddings\": false,\n",
+       "  \"torch_dtype\": \"float16\",\n",
+       "  \"transformers_version\": \"4.45.2\",\n",
+       "  \"use_cache\": true,\n",
+       "  \"vocab_size\": 91643\n",
+       "}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from transformers import AutoConfig\n",
     "# 加载配置\n",
@@ -412,10 +561,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "49021c65-54bb-4a97-a96d-b030cc3dcd13",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Test text:\n",
+      " GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
+      "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+      "The primary use of LLaMA is research on large language models, including\n",
+      "Tokenized by DNA-LLaMA tokenizer:['▁GC', 'TGA', 'CT', 'C', 'TGCC', 'AGGATGG', 'AATG', 'AAATT', 'AGGTTG', 'TTTTAATT', 'ATAATGTAA', 'AGTCAG', 'TTCTAG', 'TCAG', 'ACATAG', 'TC', 'ACATAGG', 'CA', 'AGTAAGGG', 'AAC', 'CT', 'AAAATTGC', 'TTGG', 'AAT', ',', '<0x0A>', 'KCG', 'FVGP', 'MVHL', 'KV', 'HLE', 'ADV', 'ASSC', 'RSAV', 'I', 'YL', 'TSEE', 'P', 'FEG', 'VLGL', 'RLK', 'EGI', 'AI', 'TGC', 'W', 'PRW', 'P', 'DEM', 'DER', 'SAV', 'W', 'RVE', 'PY', 'TRH', 'FG', 'RVLY', 'SFGV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n"
+     ]
+    }
+   ],
    "source": [
     "text='''GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
     "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
@@ -426,10 +587,45 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "ebf869c8-866d-4770-8f64-79d671f88663",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e497889a1c3c484cb57c4b6fd93b45ab",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some parameters are on the meta device because they were offloaded to the cpu.\n",
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'The key to life is to accept the fact that you are going to die. The key to'}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "import torch\n",
     "from transformers import pipeline\n",
@@ -448,23 +644,53 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "40a22c70-f1c4-4cd5-a118-2f5db40790e6",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTCTCCTCCTCCTCCTC'}]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "pipe(\"GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCT\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "aec95d0a-4269-4540-bf14-4ce157b9a194",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKETLK'}]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV\")"
+    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLK\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1cfab60-2820-4885-8961-0290c49dfbec",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/7-llama-instruction-ft.ipynb

@@ -184,11 +184,21 @@
     "指令微调通过在特定格式的数据集上进一步训练大模型，使其能够更好地理解和执行用户的自然语言指令。这种方法适合多任务场景，并能提升模型的交互能力和领域适应性。借助高质量的指令数据集和高效的微调技术，大模型在实际应用中的表现可以得到显著提升。"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e77f8b39-e75a-4014-a98a-bde5b2534bf1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "id": "7be8b814-42f6-4fb6-bf4b-ae23292030f6",
    "metadata": {},
-   "source": []
+   "source": [
+    "## 持续预训练 VS 指令微调"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -293,6 +303,8 @@
     "#微调\n",
     "./run_sft.sh\n",
     "\n",
+    "运行时间约3小时\n",
+    "\n",
     "#合并模型\n",
     "./merge_sft_model.sh\n",
     "\n",
@@ -325,17 +337,9 @@
     "        os.environ[var] = value"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
-   "metadata": {},
-   "source": [
-    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "054a2956-9045-4ad5-a878-1bfc84ad4ed8",
    "metadata": {},
    "outputs": [],
@@ -350,10 +354,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "63c8bf16-9576-41bc-b27c-c92ba4289cf4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 19839\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "from datasets import load_dataset\n",
     "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
@@ -362,10 +382,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "95928da3-ca64-4a17-80f4-945da395702c",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 1983\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 17856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
     "data"
@@ -373,7 +413,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "a3e65bcd-85ce-4261-8ba6-7665c4ec60e2",
    "metadata": {},
    "outputs": [],
@@ -384,10 +424,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "3d3fe49b-f48f-42b2-bc97-028e443111e4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4f060ff2029447b9bad5e2b2e40b7133",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(91644, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=91644, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
     "model"
@@ -395,7 +486,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "c54df9fe-86c4-4963-b313-b438894bf9dd",
    "metadata": {},
    "outputs": [],
@@ -424,10 +515,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "ee540cfb-1f6e-4e02-a3bc-c814e43685cb",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "example = data[\"test\"][0]\n",
     "example"
@@ -435,10 +539,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "7ee35528-7b3f-4e60-b88b-1bc3e950012b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "prompt = build_prompt(example)\n",
     "print(prompt)"
@@ -446,17 +567,116 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "8aa6f38f-3bcc-4566-8a66-a541db91e031",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['▁Below',\n",
+       " '▁is',\n",
+       " '▁an',\n",
+       " '▁instruction',\n",
+       " '▁that',\n",
+       " '▁describes',\n",
+       " '▁a',\n",
+       " '▁task',\n",
+       " '.',\n",
+       " '▁Write',\n",
+       " '▁a',\n",
+       " '▁response',\n",
+       " '▁that',\n",
+       " '▁appropri',\n",
+       " 'ately',\n",
+       " '▁comple',\n",
+       " 'tes',\n",
+       " '▁the',\n",
+       " '▁request',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Inst',\n",
+       " 'ruction',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'Det',\n",
+       " 'erm',\n",
+       " 'ine',\n",
+       " '▁core',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '▁detection',\n",
+       " '▁of',\n",
+       " '▁following',\n",
+       " '▁d',\n",
+       " 'na',\n",
+       " '▁sequence',\n",
+       " ',',\n",
+       " '▁The',\n",
+       " '▁result',\n",
+       " '▁will',\n",
+       " '▁be',\n",
+       " '▁one',\n",
+       " '▁of',\n",
+       " '▁the',\n",
+       " '▁following',\n",
+       " ':',\n",
+       " '▁Non',\n",
+       " '-',\n",
+       " 'prom',\n",
+       " 'oter',\n",
+       " ',',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Input',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'CCG',\n",
+       " 'TGCG',\n",
+       " 'ACCGG',\n",
+       " 'AAG',\n",
+       " 'TGGGGC',\n",
+       " 'GGCG',\n",
+       " 'ACCCCGG',\n",
+       " 'AAG',\n",
+       " 'TCCCC',\n",
+       " 'GCCGGG',\n",
+       " 'TGCAGC',\n",
+       " 'TTGG',\n",
+       " 'TCGG',\n",
+       " 'TTCG',\n",
+       " 'ATCGCC',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Response',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'prom',\n",
+       " 'oter']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "tokenizer.tokenize(prompt)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "11875339-4901-4912-86e5-afe8c74921d9",
    "metadata": {},
    "outputs": [],
@@ -498,10 +718,48 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "1b02644a-8b24-45aa-b22d-0f7ce2270dd9",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "real answer: promoter\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/configuration_utils.py:601: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.01` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "promoter\n"
+     ]
+    }
+   ],
    "source": [
     "input_text = format_input(data[\"test\"][0])\n",
     "\n",
@@ -562,10 +820,118 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "id": "68831e19-5a99-46d8-9f40-e8bf6957dbfc",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Donor Sites |||||||||||| Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "presicion 0.73 same 0.3\n"
+     ]
+    }
+   ],
    "source": [
     "import json\n",
     "from tqdm import tqdm\n",
@@ -598,6 +964,14 @@
     "\n",
     "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bc38f47-4a7d-43eb-abe8-db4310d280e3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/data/.ipynb_checkpoints/dna_promoter_300-checkpoint.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:490769d2870d4336ccd377292d22b3f78cfac78a4f0776043103562168c347b2
+size 28531988

04-gene-sft/data/dna_promoter_300.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:490769d2870d4336ccd377292d22b3f78cfac78a4f0776043103562168c347b2
+size 28531988

04-gene-sft/gpt2-small3-1024.json

@@ -0,0 +1,602 @@
+[
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCAGGATGCGCTGACGACCCGGCTGGCAGGCGGGTCCTCGTGGGCGAGGCGAGGGAGGCGGCGAGAGAGGAGCAATAGTTTCCCACCGCTCCCTCTCAGGCGCAGGGTCTAGAGAAGCGCGAGGGGATCTAGAGAAGCCGGAGGGGAGGAAGCGCGAGTCCGCGGCCCGCCCCGTTGCGTCCCACCCACCGCGTCCCCTCCCCTCCCCTCCCGCTGCGGGAAAAGCGGCCGCGGGCGGCGGCGCCCACTGTGGGGCGGGCGGAGCGCCGCGGGAGGCGGACGAGATGCGAGCGCGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGTCATGCCTGCAGGGAGAGAGGGCTGGGGCGCTATCTATGGTAGAAGTCAGAAGAGGCTGTCTATATGTCAGTGGGAGGAGAAAAGTGGCTCCAAAGCGGACTGTCTGTCCCTGTGCCAGAAGACATTGCTATTCAACGTTCACTCGCTTCTGCACAGGCCCACGTCACCCACAGACCGAGAACTCATCCACTAAGCGCTGCGGGTGGGCAGTACATACCGACGAAGTGCCTTCGCTACCCCTCTGGGTGTCCGTGCTCCCGCGAACCGGCTGGGGCTGCAGACGGGAAAGGAATGTCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGGGTTTGCACTAACCGATCCACTCCGCGGGGACCTCTTAGAATTCTGCTGTGGATAAGGATATCGAACTACCGAGTCACTGGACCTAAAGAGAGATTCACTGTATCATCGTCTGCCTTAGACTGTTCCGCACCCAGGAACTGGGGCTGTTGCGCCCTTAAGGTTACTTTGAAGCCAAGGTCGCAAACAGACTTCCGCATTGACGTCAGTAGCCGAACGCTGATTTTCTTAATCTAGTATTTAGGATGGGCCTCTGTCGCCTAGCCGCTATCGCAGAGTGGAGCGGGGCTGGGAGCAAAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCTCGTAAGGACCTAACTGTTCCAGCACTACACAGCTCATGGTGTCCTCAAGATTAGTACATATCAGAGGCCATGAACTTAGTCTTACGCTCACTGTGGCACTGGACGCGCGGGGGATGCGGGGTTGTTCCCAGCGATTTATGATTCGTTATTTGCCGCGTTGACTCGCCGTCTGTAGCCCCATGACAACGACATTCCTGCATTCTCTGCCTGGAGAGCGAAGTGACAATACTGAATTGAAACGGCTAGAATGTCGCTCGCTGAGGCTCCGGACCTTGGAGCGTCTAGAGTCTGGCTAC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGCCACATAAGTCGGGTCAGCAAGTCGGTGGCGAAAAGAGGGCCACGTCCCAGACTTTCTGGGAGGAGGCGGGACAAAGGGGCGGGGCGAGCGCAGCGCCCTCCGGGTGGGCGCCGTCAAGAGGCCGGGGGCGGGGCCGAGCGCGGCTGGGCGGGGCCTTGAGAGGCCGGCCGGGGGCGGGGAGGCTGGCGGGTCGGCGCGGGCCCAGCCGTGCGTGCTCACGTGACGGGTCCGCGAGGCCCAGCTCGCGCAGTCGTTCGGGTGAGCGAAGATGGCGGCCGAGAGGGAACCTCCTCCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATAGGATGCGTCTGTACTCTGATTAACAAACTGCTGAAGGAATAAATATGTACTTGCTGGGCAGCGCCGCCGGCCGAATGGAGATAAGCCTATGCAGCTTCGTGCGCGGCTCCCCAGCCCTTTGCTGCGCCGCGAGCTGCGCCCTGAGACCCCCGCCTCGCTGCCAGCTACTTACCTGCCCCGGCGGAGGGGGCCATGTTGCTACACCTAGGCAGGCGGCAAGAAAGCACGCGTAATGAATTCCTTATATCCCCCGCGCCCCAACGGCGGCGGCGCGCCGGCCGGCATGGAGCCCCGCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGGGTAAAAAAGGGGACAAATGGAAATTTGTGTTGCAGATAGATGAGGAGCCAACAAAAAAGAGCCTCAGGATCCAGCACACATTATCACAAACTTAGTGTCCATCCATCACTGCTGACCCTCTCCGGACCTGACTCCACCCCTGAGGGACACAGGTCAGCCTTGACCAATGACTTTTAAGTACCATGGAGAACAGGGGGCCAGAACTTCGGCAGTAAAGAATAAAAGGCCAGACAGAGAGGCAGCAGCACATATCTGCTTCCGACACAGCTGCAATCACTAGCAAGCTCTCAGGCCTGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCTGGGTCCTAGAGGCTACTGAGACAAATGCTGGCTGAAGCCGAGGTACCTAGATTTGTGCAAATCGAGGAGTTCTATTCTGCGTGTTTGTGAAAGGGGGTTAGACGAGAGGAGTGCGGTGGAGGCGAAGATAATAGGATTATTCTTACAGTGTAATAATAATTATTGGTGCTAATGTTTAGGAAGCCGTGAAAGATAACACGAGGGGAGAAAGTGGATAGAAAAACCACCGCTGGCAGTTCTTTTCATTTGAAAGTGAATTGGGAAAAAGCGAATAAAAATTAGTCATGAGTTTTGGTA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTCGTTGCGTAATCCTCCTTCCGCACCAACGCAAGGCAGGACAGGAGGTCAAATGAGTTCGAGATCAGGCGCCACAGACTCCACTAAAGCACAGACACCAGGTGCGGTAAGACACGCTGGCTCCAAAGTAGGACACCAAACCATTGCTAATGCCTGGGGAGGGCCAAACATGGGCAGGCTGGAGCACTCTCCTCAAAGAGGACCACAAGCCGTGACAGCAGAGCCCCTCGGACTGATGGGACACGGCTTAAACTGAAGTTCGAGGACTCTGTTCTGCAGAGGTCTGAGGCTGGAAGGGGG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGGAGAGAGCTGGAGTCTCGCGGCCTCGGAGGCCGGGAAGCCAGCCCCGGCGGGGACCCGGGCGGGGCGGAGCCAGCGGAGGCCCCACCCCCGGCGTCACCGGCCCCCAGGGGGGCGTCGCCCCCACCCCGCGCTCCAGGTAGCGGCGGCCTCAGCTGCCGGCGGTGGCTGGCTGAGCTCCCCAGCGGCCCGCGGGCCGGGGCGGGGGCGGGCCGGGGGCGTTCCCGCGGGCCACCGCCCGTGATGTCACAATCGCGGCGGGCCGCGGCGCTCCGGGGTCGGCGGTGGGCGGCGGGCCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGACTATACCACTTGCCGGCCGCGCAGAGCCCTCCCCAGGTCGCGCAAGGAGCTCGAGCTACTAACGGAGGAGCGGCGCAGGCCCCGCCCTTAGTTCATGAGCTTACTTGTACAGATCTCGGCCCCGGCCCCGGCCCCGCCCCCTGCCTCATCGTGGACCATCACGCTTCCAGCCCAGTCGGCTACGGAGATAATCCATGGAGCTCCGAGGCGCCGAGAGAGCAACCTTAAAGTATCGGGATCAGCGACATTATCAGCGCTGGTCCCTGCAGCTTCTGGTTCGAGATCTGGAAGGGCGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGTTTAAAAATGAACCCCCAACAGAGCCACCCGCCGGGGACGGTTCTGTTTTCAGCACAACCACGGGGTTCGTTTAGCCTGTCGCCCAAGTTTGTTTCCTCTAGGTCGCCCACCTTCCGTCGAATTCTGTCACTAGACTTTTATACTGGGTTTGTATGTAGCTCGATTCTGCATACAAGCTGGACTGAGAACTGGCAGGAGGCGAAGATGAGGGGGCGGCATCCAATTTGTTGTGATCCTTTTGCGTCAGGCTTCTGCCTGAGCTCGGTGAGGTCAAGCCTCCTCTGCTTCCACCCCTC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCCGCTCCGCGGGGCAGGCGTCGTGGGCCCAAGGAGGCCAGGCCAGGAGTCTCGGCCTCGGGGTTACGCGGCGGCGGGCAGCTCTTCCGCTCCGCTCAGGGTGCTGCTAACCCACAGCCAAGCAGCGACCCGCAGGACCGGCGAGAACCCCGCCACTGACTCCGGCCGGAAGAGAACGTGCAGAGATGTACGTCACTTCCGGGGCGTGGCGTGGCCGGGCGTGGCCGTCCCGCCCCTGGCGGCGTGCCTGGTGCCCTGTTCCGCGTCTGTGCGACCGTCCGTCCCGAGCGCGCAGCCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TACCGGTGCTACTTTACGCTGCGTGTTCCCTAGCAGGGGACCTAGTGTGCTACCATAAAAGCACGACGATAAGACTATACCTCTTCGCCTGCTACTAACATAGGGCACTTCTTCCCCCTCAATCTATCGACGTTCGTTAATATCACCGACTCCCAGGGAGGATCTAATAAAAGGAGGAGGGCTCCTGGAATAATAGAGGAGTGGTGCTGCGAGGGGAAGGAAAAGCCAATCCGTGACCCCTTTCCCGGAAAGCTTCGAACCGACAGCCAGAAGCCCGCACGCACCTCGCACCATGAGATG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCGAGAAGACCTGGGCTACTGAATGACGAATGGGACGCACACGTACTGGCTACTTTTGTGGAAGATGGTCCGCCCTGGTTTGGATTGCGCTTTGCCACGAGAACCGCCGGGAAGGTGGGGGACGGGGACGGGAGCAGCCAATCACGCGCCGTCCCGTTCGCTTCGTCAGGGCGCGATCGCGGATTCGGCACTCAGTGTATGGTACTTCGGGCGAGCGAGGAGTCGCGAGCGCTGTGAGTTTCTCCTGGAGTTTTCTGCCGTTTTCGGACCCTAAAGCTTATCTAGGCTCGCGCTCTGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCGTCCTGCTCCAATTGCCAGGAGATACCATTAGCGGACTGACTCCAAGAAAAGAGATCCCTTGAAAAAGGCAGTGCTGAACGCGAGTATGGCGCTGCTTGCGGGCCATTCTGTGCGCTACGCTACCCGCTCGTATTAATCAACAACAGGACCGGACTCTGGTTGCTTGGGAGTTCAGGTGGACAACAGCAACTACTAACAAGGTTTTTCGTTCATAGTGCGGGAATGGCAAAACACGAGAGAAACGCACTCTTCATAGGTCACTATCCGTCTGGAAGAAGGCTTTGGACAGAACAGGG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTGCCCCCAGGCACTGACAGGGTAGCAGAGGGAAGACACATGAGCTGCTTTTTAATTTTTTTTAGAATTAATAGAGACGGGGTCTCACTCTGTCACTATGTTGCCCAGGCTAGTTTTGAACTCCTGGCCGCAAGCAGTCCTCCTACCTCAGCCTCCCAAAATGCTGGGATTACAGGTGTGAGCCACCGTGCTTAGCCATGAATTGATTTTAAAACCTTAAATGAGGGGCCAGGTGTTGTCTTTCCGTTTGCCTGAGTGACGCGGGTCTTCTCCAGGACATCCGGCCACAGATCTGCC",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CAGCCCCTATGTTGGTGTGGAGACCCTCACCCGAGCTGGGCCTGCACCCTACACCAATCCGAGGTACCACGATCGGATTACTACAACAGCGGGGTCAGCTCAGGCGCTCGGTCCCCCGGGATCCTGCTCCCAGGCACACGATTCCGTTCGTTATAATTATGTAGACGGGGCCTGCGGTAGACCGACTGCGGGTGACCGAGTAGTGCAAAGTCTGGCTTCTCTTTTGATAGAGCACATCGATCTGTCTTCCTCTTTTCACCGTCGATGAGCCCCGCGTGCCTGGGACTGCTCCAGGCTCCA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCGCCGCCCACATTAGTGTGGGGCCCTGCGGCCTAGCGTCCCTCACCAGAGGCCTCCCCTTGCCTAGCTGGACCGCCGAGGGACATCGACGAGTATCCTCCTCCTGCTGTCCCCGGCTTCGCCTGCCGCCCCTAACCGGCCAGTCAAGATGGCCGCCGCTGGGTGAGGCAAGCTGGCGCGCCGCGGGGGCGTCTGGGAGTTGTAGTTCGGGACGGCGGGCTGACGCACTTCGCCGCCGGCCGACGGGCGCCATTGTGCGGCGCGCGCCGGGTGAGTGCCGCGCGAAACCTGCGTCCGTC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCGGGTAGCGATGGGCCGGGCCTCAGTTGTTTAAAGGCCCGATGCGACCCCGCTTAAGACGTCAGATCGTTCACCCCCGAGGGTCGCCGGATTTCGCTCCCGCGCGAGATGGCCTCCACCAGAGACTGCTGCTGCGCCTTGGAGCGGTGCCTCGCCTCCCAATCCTCGTCGTGAATTGCCCCAATTGCTGATCTGGGAACAAGCTCGTGCCAGGCGGGCGGAGAGTACGAACTCCTTATAGACCAGGATGGGCGTAGCAAGAGCGCGGGCGGCCCATGAATGTCTGCAGCGGTGAGGACC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGATGCGGGGCCGGAGGGGATGGAAGGGGCCGATCTAACACGATATTCCGTTCAAGAACTACAACCGTCTACAGCACACCATAGCCGCTCGCTGAAGGATGTCGAAGGGCGCCCAGGCACCCACCACGTTTGGCCCCTGCCCGGGCCGGTTCCGATATGACTCACTCATTTACTCCCCTAGAGCTCCGAGCCTCGGATCGCCGAGGTGACCCTTACCCGCCTTCCTACCACCTTAGGGGATTTGGCCCGGAGAACGAGATCACCCTCTCATCGTGGCGGGGTATTCCCTTTAAGGTTTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCCGTCGGTCCGCGGGGCTCTCGCTGGAGCCTCCCCGACTCCGGTTTCCCCTCGTCCAGAAGCCCGACGTAACCAAAGCCCAGTCTGTCACTTTAAACACGCCCCGCCCCGCCTCCCGCGGCTGTGTTGCCTCCTCGCTGGAGAACACCCTGGTCGACCTCTGTGCGTCCGTGTGCGCGAGCGCGTCCCGCCGAGGCGGTGGGCAGGGCGGACGGTGCGCAGTGCGTTCCCGCTGGTCGGAGCCAGCACACTAACCACGCCACGCGCCCTGCCGTCCCTTCGCCTCCAGCCGCTGCAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACCTACTCGGGGTCCGAGGAGGGAGGATTGCCTAGGCCCAGGACTTTGAGAACACGCCTCTACAGAGATTTACATTTTAAGAAAATTAGCTGAGTTCGGGTCGGGCGCAGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCTGGCGGATCACCTGAGGTCAGGAATTCGAGACCAGCCTGACCAAAACGGCGAAACCCGTTCTCTACTAAAAATACAAACACTAGCCGGGCGTGGTGGCAGTCGCCTGTAATCCCAGCTACTCCAGAAGCTGAGGCAGGAGAATCGCTT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCGGCCGGTGTGCGGGACACACTATTGTAATCCTAAAAGGCCATGGGCCCCTGCTGGGGTCAAGAATAATGCTCAAAGTTTAAACTACCGAGTGTTGCTGAGGCTTGGGTTGCACATCGAGAGTCGCGATAACACTCACCCAGGGGAACATACCGCGAGGGATTACGGACGATGGAGAGATCACCCAAGGCAGTGGGAGTCCGCGCTTCCTCCCTTAAGTACCGCTGGTTGTCCCCCGACGTAGTGTAGATAGTGAGCCCTCTTGTCCGCTGGCACGTGATGTATGTGTGTCCTTGTCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGGTTAGGCGCGTGCCGCGAGAACAGAGTGGACGGAGCGTAGGAGAGACCGAAAAGGCTGGGGGTGGGAGTAGCGGATTTGAAGCACTTGTTGGCCTACAGAGGTGTGGCAAGCAGAGCACCTCAGAACTCAGGCGTACTGCCCGCCGCCCGAGCCCTGCGAGGGCCGATAGCGAGGGTGTGGCCCTTATCTGCACCCAGCAGAGCGCCGGCGGGGTACGGTCTTAGGACCTCGATCTCCTTCTCCCTCATTTTCTCTCATCCCTACCTATTGTGGGTGAGTCCTGGCCCCTGGACGGGG",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTTGCATTAATACACCCGGCGCCAGAGGAACGTGGGGATCAATCATATTACACTGCTTCAGACTTGTAATTTTATTTGGTTCAGGCATCGTAGTGGTGAGGCTTGGCAGAGAGCCCGAGATGGAGCTATTGAGACGTTTCAGGCTGGGCGGATAATCGCATCAGTAGGGACTTGGCTAGATCTCCCAAGCCTTCTCCACTCTTCAAACTTTCACCAAACTCTTAGGACCCGTCGGGTCGGTTTCATAGACTCAGCCGCTCAACCTTTTAAGCGAGCGGCGCGGCGGGGCGGGCGAACTCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGAGCGTCATCTTATCATTCTCTCTTTTTTTTTTTTTTCTGGTAATGATTTTTTTTGTTTTTTTTTTTTTTGATATTGACAAAAGTTTAATCATTTCAATTAAAAATGCCACTAATTTGACTTTTTAAGTAAAAAATGTAGGGGGTTTTAAAACTACTTTCCTACTACCAAAAAATCAGAAAGTATCTAGCTTTCTAAATTGGGAAAGCAAGCAATGTTATAAAAACACTGAAGGAATCTCTTTCTTCGTGACCTTTTGTTAAACTCGGTTTAAGCTGTAGACCTTATTTAAAATAAAAT",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGCTGTGAAACAGGGCCACCTGCAACCGTGGCTTTTCAAAGCTGAAGCCATACCCTGGAGGGAGAGAGGATCCTTGGGCACTGACCCTGGCTGTCCACAGCTAAATGATCATGGGACACCCATGACCATGGGCGTGGCTGGAGGAGGCCCTGCCCTCAGGGGTCTATTGGCATCCAGGGGAAGTGTCTACGGGGTAATGATTAACCCACACCTGCCACCTGGTTTTCAGCCTTCTTGCCCTGCTGACTCATTTTGCAAATCCCACTTGCAAAATTTGGGAGCAGCCTGGGTCCAGCCCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTGTGACAATCATGGAGCAGTTGATTGACCCCTTTTCGAGCGCTCGCTATGCTTTAATTTCAAGACCCGCTCTCTAAGGTGCTCTTCCCCTTTAAGACTCACGGCTCTTCCCGTCAGGCTAGCTCGACCTGAGGATTCGTCAACCTCCTTGACAGCTTCCGCTGGCCTGCACCCGAGATTGGGGACGGTTTTCCGCACATCGTGCCTTCATTAGTTAGGTCCTGCGAGTGCATCCAAGGGCCCTCCCCCAAGCCAACAGCAGAACCAGCTCAATTTCTACCCTCACGGCGCTCCGCGGTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TAGAGAGGCCAGAATCCCTCGGGAGTGCTACCCGGTCAAAAAATCGCGCCGCCCCGTGGGGGACCGAACTTAGCGATGACGGTTCGTCTAATCCCGTAGGAGAGTATCGGGCGAGCTCGGATTGCGAGATTACCTAGTGTTAGTGCGACCGCCCAGGCACATCTGCAGACTGACGTCAGTATGCAAATCAGAGTCGGAAAAGCTGCCAAAATTTTCTCCTGCTGCGCCACCTAACCTGCGTTCGGGGGTAAACCCATCTAGGGCTACTTCAACAAAACTTTGCTGCCCTTCCTGCTCCTC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGCTTTGTTTAGGTCCGGAAGGCGGGCTTTCCTGGGAGTGGGTGGGGAGGGGGCGTTGATTCTTGACCAATCCTTTCAGTCCGTTGGGTGGTGACCAGCCAATGGGCCGGATGGATAGGACGCTCCTCCCGGAGAGTAGTGAGACCCCTGGTGCGGGGCGATTGGCGGCGGGAGCGATGAGTGGCAGCCGCACGGCCCAACGGGAGCTGTGCGTGGGCCGCGGGGCGGGGCCAGGGCGGGTGCGCGGCGGCGGCGGGGTGGCTGGGCCGGCGGCGGCGGCGGTACGAGGCGCGCGCTCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCATCCAGCGAGAGGGGCAGGTTCCGCATTTTCTCTTCCCCTTTCCCAGCGCTTCCTCCAGCACCCGAAGCCCCAACCCTGCGGGTCAGGAACTCCCTAGTCCCCAAGTCTAGGGATGAGATGGGGGAAGGAGAGCCGTCAGGGTTGACCTGGAGTTTTGTCCGCTCCTCCCCTACAGTGATCCCTCTAGCCTTCTCCAGTCGCCTCCGCCATGTCCGAGGAGCTGGCCCAGGGCCCCAAGGAGAGCCCCCCGGCGCCGCGTGCGGGCCCCAGGGAGGTGTGGAAGAAGGGTGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACATCGTATTGCGCTTACTCGGGTCGTGCAATGCACTGACTAGGAAAGGGCTGCGGATGCCTACGTACATAATGCTGTTGCGCACCTATGCACGCGGGTCTGGTCCTGGGCAAGAACCGCCCCCTCTCCGGGCCTGGTTGCCTTTTGATTTGCAGAACAACGGGCCAGGCCCCTTCCCTCTACCGCTCACAAGCGTACATCGCGACCATACCTGGTCCAATGCGCTACGGACGAGGGCAATTCTCGTAATAGGACCCACAGCAAGTCTTGTATCACCTTTGGCTAGCCTGTGCCGGCTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACGTGACCGACGTCTTCCGCATACCCGGGGCTTCCCTCACTTCCAGGACAACCTGTTGCGCCTCTATCTCCTCCCCTCCCCGCAGTTTCCCCGCCTTGGCCTCTAATAGTAGTGAGAGCACTTTAGGCGGTATAAAGTCTGACGCTAGAATCAATCTAGCCGCGGCGATTGGCCACCTGGGGAATCGTTAGCAAAATGTCACGTTTACTACTTTACGGCGATCTTCCAGGGTCCGGGGACTGTGCCAAGGATCCTACCGGGGCTGGCAGGAGCCTAAGACAAATACGCTAGCGGCGGAGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAGATTTCACGCCTAGCGGCTCAGGATCGTGTGTGCTACAAAAATAGGGCCGCAGGGCCGTATAGTCGCACGAACATACAGGTCGTCCGATTCAGGCTCGGGGGCTTGGACGGAGATGACGGCCAACTCTCACCCTCACCCCTCATTTAGGCGGGCGCCGGGGGCATCATCGGGGTCCCTGGGCCCTCCCCCTGCTCAGTGCGTAGCTCCGATAGAAAAGCTAGGGCATACGTGCAGAGACGGGTTGTCAGGCAGACACCCTGTCTGTTGGCAGCTAGCAGCTGGCGGACGCGACCCGGA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAACCATGCAATCCGGATGTGGCAAACAAAGGGTGGGAGCCCACTTGTCATGGCATTAATTGCACGTCTGCTTAGGGCGAGTTTTAACCGTGGCACACAAAACAGCACGATCGAGGATACGGGAGGTCGCCGCGTGATATAACGTCTACCCATGGTTCTCATCGAACCTCTACCTAGACACTGACCAAACCAACGCTCCCAGTGCGCACGGCCATTTCTCATAGATATCTTGAGCAGGTTGCACCACCAAGATAGTGAGCAGGGCGGTGGGTGGTGCGGAGTGCCGAGCGGCCTCACCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCGGAGGTCGGCGGCCAGAAGCCAGATTCCCTTGGCCCCAGCTCGTCCACTTCCATCCAGCAGTAGCCAAACAACCGTTGAAAATGGCCTGGCGGCGGCCGGGAAACACCAGAGAACTATCTTCCAACCCCTAGATCCCGCCCATCTCGTTCCGGCGGTGTTTCCGTGGCGACGCTATCCGAAGTGCGGCTGCGCAAGGGTGACGGCGCGCGAGCAAGGGGGAGGGGGTGTTTTGGTTCTAGCCGCTCGCCGTCCTTGCAGGCTCTGCCGTCGGAAAGCCGCTCATTCTCGCTTCCCCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTGGTTGTAGGTGGGAAGAATCCTGGGTTCCCATGCTGTCTGAACCAGAGTCATTTGTGGCCCTGCCCTCCCCTGGGACTCAATTTCCCCACCTATAAAATAAGCCCCAGTGTCCGCGAACCCTGGAGGGGCCCGCACCACTGCAGGAGCGGCCGCCGGCGCCAGGGGGCGCCTCCTCGATAACTCGGCGCTCGGCTGGCCAGGCACCGGCGCGTCGGCCGCTCGATTGGTCGGGTCGGGGCCGGCCTGAGCGCCGCGGGCCTGCGCCATTGAGGAGCGGCGGGGAGGAAACGCCGCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TAGGAGGCCTGTCTGCACAGCTGTGCTGGGAAGCTGATCCATTCCAACCACCGCCTCATGGCCGCTTACGTGCAGCTCATGCCTGCCCTGGTACAGCGCCGCATCGCAGACTACGAGGCTGCCTCGGCTGTGCCAGGCGTTGCTGCTGAACAGCCTGGGGTCTCTCCATCAGGCAGCTAGCCATACCCAACCCCAGGAAGGAAGGCCTTGGATGGACCCTCAGATTGAAGGACCCGGTGGACCTTGGGGTTGGTGAATCCTAAACAGAGAGAATTCGAGGTTGCCTGAAAGCTGGGTGTC",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTTCTAGCTATCTTGAAATACGCAATACTTTGACATTAACACACTGCTATGTTCTCTAATGACATCCCTGGAAACCTCCTAGGGGCAGCCAGATCTTTCATGATAGTGGTTGTCAGTCCTCATATGGAGGGTGGAGGTTTGAAGCAGAGAGCCAAGGGAGGTTTTGTGCACCTATGCTTGTTGTGTTTGTACACAATGACTATGCATACGCTGTGAGTATAAAAGGCTCATTTAATCCTATTGTGTCCCAGGCTTGGTTTGTTTTCAAATCATTACAGCATGAATTTAGAAGGTTTTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCGAAACCCCTCATCTTGGGGGGCCCACGAGACCTCTGAGACAGGAACTGCGAAATGCTCACGAGATTAGGACACGCGCCAAGGCGGGGGCAGGGAGCTGCGAGCGCTGGGGACGCAGCCGGGCGGCCGCAGAAGCGCCCAGGCCCGCGCGCCACCCCTCTGGCGCCACCGTGGTTGAGCCCGTGACGTTTACACTCATTCATAAAACGCTTGTTATAAAAGCAGTGGCTGCGGCGCCTCGTACTCCAACCGCATCTGCAGCGAGCATCTGAGAAGCCAAGACTGAGCCGGCGGCCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCATCTTAGAGTTTCATACAAGGATCAGATTTGTGCTATCGGAGGCCGCCGATCACCCAAGTAAGCTTCATACGGGGAGGGAGGGAGGGCGCGAGCTTGGGAGTGGAGAGGGAAGGAGCAAACTTTGGCATGTATCACAGGGTAGCTCTGCCCCTGCGCGCCCCCTCCAGATCCGGCCCGCGGCTCCCCTCCCGCGGGACATGCTGCGGGGAAGCACGGCGAACAGCCCCTCCTATGATCACAAATGGCTCACCGCTCTCGGAGCCCGGCCGCCACCGCCTGGTGTGCGAGGCTATCCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCTCTGACTCGGTTTCCCCTCTCCCCCGGCTCCGCGGTCGCCCTCCTCCTGGACTGCTTATTTCGTCCTCGGCCACTGCCTCTCGGGCTCGAGCTTTCTGCTTCAGAGCAGGAGAGAAACGGAGCGAAAGCAGTTTCTGTCTCCCCGGGGTCTGACTCGGTCCCCCGCGCTCGGTTCTCTGTCCCCTCCCCCTCCCGCCATTGTTCCCGGCAGGAGGAGTGCGCGGCGCAAACTTGCGAGTTCCCCCAGTCTTTGCCCCCGCAGGGCCGCGGACGGAGGTGGTGGCGGAGTTCCCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGCATGATGTTGGCCGACTTGTGCTAAGGTCCTTGGAGAGGTGGAGGCTCTACGGGGGGTGGGCAGTACCCCTTAGAAAACACCCCTCACTCTCAACAACAGAGGCGGTCTGACGTATTGGAACTCATTCTTTAAGGCCAGAGCTGTGGTTGCTCCAGCTGCAATGTGCGCTCTGATGATACTTTATAATAGATTTGAAAAGGGGAGGGACAAGCCTAAGGTCATGGCAGGAGGCTGGGCCAGGGAAGTGTGGGATCCCGTGGTTGTAGTCGGCGAACCGAACGTCTCGGCCTCTCTCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGGGGGTTTTTTGGGGGCTCGGGCGTCGACTGTGGGGCGCTGCGGAAGGGTGGGTCGTCCGCGTGGGGTCCTGGAGCGCAGGAGGCGGAGGGGAAGCCGCGCAGGGGCCACTAGGGAGGGGCTCTGGCGGGGTCGCAGGGGCGCGGTCTGCGGGCGTGGGGGCGGGACCTCGGGGGCGGGGCCTGAGGGCGCGCGCACGCCTCCGGCGCGCCCCCTCCCGGCCGCCATGTTGGCTGGTGTGTGGGTGTCAAACTGAGCCAGACGCGGCGGTGGCGGCGGCTCCGCGGGCTACGGTCGCTC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGAGAGGGGCGGTACGCACCACGGGGGAAGCCAATGAGAAAATCAGGCCCAGCCCTAGGGGGCGGTGCTGTCGGTCACATGCGCACCTGGGGCGGGTGGTGGCGGCGGCGCGGGCACCGCGAGCCGGCGGAAGGGAGAGGGGCCGGCCTGGGGCGGGGTTAGGCAGGTGAGTGACAGGCTCCAGGGGGCCGGCCCCTGCCTGGTGCCCCGAGCGAGCCGGGAGTAGCTGCGGCGGTGCCCGCCCCCTCTCTCCGCCCCTCCAGCGGAGCTGGTCTCCGGCCGGGCACCGTCGCGGGCCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGACAACCCGCCCTTGTTCCCGGTAGAGGAACAAAGAGTGCTGATTTACGAATCTAGGCTGCCAGATTCTGTGTTTCCACTGGAGGGCGCGTCCGATGCAGATGTTGGCTCAAATTCCATCTTAACCTATAAACTCAGTTCTAGCGAATACTTCGGGCTAGATGTGAAAATAAACAGTGATGACAATAAACAAATTGGGCTCTTATTAAAGAAATCCTTGGACAGAGAGGAAGCTCCTGCACACAACTTATTCCTGACAGCCACAGATGGGGGCAAACCTGAGCTCACAGGCACTGTTCA",
+        "output": "promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TTCCAGCACTCCGGGAGCCCGATTGCGAGTTTGTGAGAGTCCTTGGTGGAGGAGGTTGGAAAGCGTTCTCCCTCCCACAAAACCCGGTGCAGAGGGAGGGAGAGCCAGGTCCGTAGCGTGGGTCCAAGAAACTATGGATCCTGGAAGTGAGCACACTTTTGTGTCTTAAAAAGTTTGGTGTCTCGTTTTGAGGTGAAAAATGAATGAGATAAGTGCTTTTTAAAAAGGTAAAGTTTGTCTTACAACTGAAAAAGCTTTTAAACAAAACTATAGTTTTTGCGAGTAACTCCCGTGACACCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATCAGCCCTGGCTAGGACCAGCTTCCCACTAAATCGCTTGCGGCAAGGAAGTCTCAGCCCCCGAGGCAGTCCTCACTGTGAGCAAACCAAAAGATGGGATCACTTGGGGCGCGTACGGGCTCCCAGCGCTGGCAGAAACTGTAGCGTTGTAGTAGCCTTTGTGACTGGTTGCCTGTTGGAAGTCTACCCCACCCCCAAACCATAAATCCACCGGAAACTCCTCCAGGACACCCTCTCAAGCAAGCCCCTGGGATGTAACTGCCACAGGGGCTGCGCCACGATGTTCCCTGGCTCTCAGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACCGCGTGAGGATGTGCCGGGTGGTCCTTTCCTCCTCCTCTTCCTCCTCCTCCCGGCTCCCTGCCTAGTCTCCATATAAAAGCGGCGCCGCCTCCCCGCCCTCTCTCACTCCCCGCTCCTCTCCGCCGCGCACTCTCCGCGGCGCTGGGAGAGGGCGGAGGGGGAGGCGGCGCGCGGCGCCAGAGGAGGGGGGACGCAGGGGGCGGAGCGGAGACAGTACCTTCGGAGATAATCCTTTCTCCTGCCGCAGTGGAGAGGAGCGGCCGGAGCGAGACACTTCGCCGAGGCACAGCAGCCGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCAGGAGTCGGTAGGGGCAAGAACGTTTCCGGCTCCTCAACGGGAGAAGCGCGGGAAATTTGTTGACGTGGAATAGCTCGACGAAATCATCGGCAGGCGCCCGCCAGGGAGCTAGCAACGGAGCATATGCGACCGTCGGTTCGCAAGAGACACACCTCGGGCCTTGCCCCGGAACGCCGCAATGGCTCCTACCTTTCCGCCGAGTGCAGGTTGATGCTGCACACCCAGAACACTTTGCGGCACGGGGGGGGTTATATGACATGCAAAGCCCTGATTCAACTTGGAGATGAGCGGGTCTGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAACTCCTGAACTCAGGTGATCCGCTTGCCTCGGCCTCCCAAAATGCTGGGATTACAGGCGTGAGCCACTGCGCCGGCATCCTGGATGCAACACGATCAACGAATGATTGCCCTTTATTCTGCCTGGTCTCCCAGCGGTGCTGTTGTGTCTTCATTTTAATGGATTTTAATTGATGTCACTTAGGGGGGCCCTCAAGCCTTCCTTCCTCTATCCATTTACGTGGTGTACCATCGAACTTTATCGACTCAGAAGTCGTGGGCGGAGGGCGTTCCGATAGAACCAATTGGCTCACGTCTATA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAGATGGCCTGAGTCAGGAGCAGGGGCAGAGCTCAGGTTGTCGTGGGGTCCCGCTGCCCACGTCAGACTGGAGGTGAGGGATGGGCGGGGCCTGACAGCAGGCCTGGAAGGAACAGGATGTCTATGCTGGAGATAGAGGGAGAGGACAGTGCCAAAACCCAGCTCCTGGCCAGTCCCCAGCTCCTCCCTGCCTGGCCCTATCCCAGGATCCCCTCCCCGGCCTCCCAGCTATGATCTACCCCGGGGCCCAGACTTCAGGCGCCTTCACGATGCCGGCGGTCAGTGGTCCAGGTCCCTTAT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCTGACACCCTTCGGCCATTTTTTTTAAACGTGTTGGGGTTTGGGTTTCAGCCACAGGCCTAGCACATGGAACCAAAACGCTACCACGGACTATGGAGTTGTGAACCGCGGAACAGCCGGGCGGACTCCGGGGCTCCTGTGTTTGAGATACGCGTGCCATGAACCTCCGCGCCGGAGGCGAGCCACTGGTAATTTACTTTCCAGTTGCGCCAATCCGAGTCTACCTTGATATTGAAAGGTAAGTCAAGAACAAAATTATCGCATAAGACTGTTGAGGTTACCCCGCGGCGGGGTCGGGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTGAGGCTGGCAGGCACTGGGCAGGCAATGAGTTCTTTCCCTGTAAGTTGGGCACAAAAAGATCGGCATGGGAGCAGCCTCCTTTGAGACAGCTGCTCTGAGAGAATGCAATAAGCAGGGAGCAGCCAGCAATTCCTCCTAGCAGAGGGCGACTCGTGGGAGGAGTTCAGTTTGCCAAGTATTGTCATTTGTTGAGAGAAGGTGTGTGCTCAAGGAGGAGTTTTAACCTGGAGGATCATTAACTCTTTTAGTCAGCTGAGGAGCTGCGGTGGCTCGGCGAGTTGGAGTTCATCCTGGAAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCGCGCCCCTCCCCCCGGGTCGCGCGCCGCGGGCCCGAGCGCGAGACGCCGCTGCTCCCGCCCCCGCCGCCTCCTGGAGTCGCCGGGGCGGACGCGCAGTCCATGGGGCGCGGTGGGCCGGGGAGTTGCCCCAGGGGCCGCGGGAGTTGCTGAGAGGAGACAGGGTTGGGCTTTCTCCTCGCCCAGACCCCACCCCACCCTGCCGACCCCACCCCCTGCTCCTTCCTCCCCGGGGGCGCGCACTCGGGCACGCGCTCGGAAGTCGGGGGTCGGCGCGGAGTGCAGGCTGCTCCCGGGGTA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATCTCCCCTCCTCGGTCCCTCTCCCTCCCTTCCCCCTCCCTTCCCCCGCCCTCCCCCTCTTTTTGCTCCTGCTCCCCCCCCCCACCCCGCCTTTCTCCTTTTGCAAGAAAATAATTTGACAGTCGATTTGCTGACAAGGGAGGAATTTGCATCCTGGATTTAAAAAAAAAAAAAAGGCCGAGAGGAGCTTGGGAACGGTTGCTAGGGGTGGGTAATGGGTGAAAAAAGGGGGGTACCGGGGAGCGGATAAGGAGGGTTAAGGGAGGGGGCGAGGATGGGGAGCAATGCAAAGGTAAGGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCAGTAAGTCGGGGGGGAGGGAGCCGGGGCCTGGCCCCGCCCTATGGGGAGGCACGGGTGGCGAGGTCGGCCGTCCCATGCCCCGCCTGCTCCAGGTGCCGCCGTCCCCACAGGTGCCCGCCCCAGGCCCGCTGGAGCAGCCTGTGGCACGTGGGGTAAGTGGAGGCCGCGGCGGAGCCGTGCGTCCGACGGTTCTGGGGCGGGGGTCACCGGGGCCAGGCCCGCCAGGCCCTTACCTCACCAGGCCGCCTCCGCTGTCGCCCAGTCCCGGCCGCTGGCGGGAACTGACCTGGAGCAA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCTACCGCTGCCGGAAGAGCGCGGCGCCCGACGGAGCCGTGTGGAGGCCAAAACTCCTCCCGGAAGCCGCTACTGGCCCCGCTTGCCAGGCCCAGCGTCTTTTCTGCATAGGACCCGGGGGAAGCCGGGAAGCCGTTAGGGGGCGGGGCAAGCGGGCGGGCGTGCGTCGGTCAAGTTTCCCGGGTCGCGTCAGGCTTCTTTCTCAGCACCGAGCGAGCGACGTTCGTGAAGCTTTCGTTTTGAGCGGCCAGACTCTGCTGTTCTCAAGCCTGGGAGGCACAGAGAAGCAAGACTTCCGCA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GAGAATCCCTCTATAATATCCTTGCACAGTTCCTCCGCAAGCACAATATGATTAATTTTTACTTGGCTGATGCCGAGATCTTAACTGCCCTAATCCTCGGCTCGAGTCGAGTTGGGGTTTCCTAAATGAATTACTATTGCAAATCTGGTTATGGCGCTCCACGAGATTATGGCAAAGGCTGAAGAATTTATTGACAGATGTGTTGTTAGACTGTGCTACGTTTTAGTGGCAGGTAGTGGAGCAGCCACCGGCGGACTCGAGAACGCAGTAACATGGCTCAAAAATGGTAGTATTTGTTTT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGGGAACGCACCTGCGTCTAGACGCTGACGCCGTGCGCGGGGCGGGGGCCGGCGAGCATGCGCAGTGTGGTGTGGACAGGGCGTGGTTGGCAAGCATGCGCAATGTGGGCCGGGCGGGGCTGGCGAGCATGCGCAGTATGGGGCGGGGCGGGGCAGGGCTGGTGAGCAAGCCCGCGGGGAGCGAGCGCGGAGGCGGGCGTTGCTACGGCAACGGTGGGCGGCCAGGGAGTTGCCTGAGGGAAAAGGGGCAGACGTCCCTGGGTTCCGGTGTTCGCGGAGGAGTCGAGGCACGGAGAGGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTTCTGATTCTCGGTGAAAACGGTAAACATCGGCGTGCTATGTATGGGGATAGCAGGAGCTCTAGCATTTTTGCGCTGGGAAGATTATTCTTCTCTATCCGTCTATGCGAGCCGGCGTTTCGCAGGGGTACCCCCGGGGGCACTGCGTGGGAGGCCAGGAGGTTTTAGAATGGTCACCCGCCGGTTAGCTCCTTGGTAGTATTCTCGATGGACTGCGCGTTTGAGTCTTAAGGGCGTTTTCACGCTTCCGCCTTCTGGAAGCCTGCACTCCGTTTAGAGGAGCCAGAGACCCGTCCCGGT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTATTGGGGATGCAGGGCTCCGCGGTAGTTGACCTAGCAGGGCTAGTCACATCACGCCATAGGGAGCAGCAAGGGGCGCAGCAGTCAGCACGGCCGCTATTCAGATACGGTAACCAGACGCCGTGCGGCGTGCGCACCTGGCATCCAATGGCTTTGGGGATAGGGGCCGCCTGTGGGTTGTGGGTACGAATTAACGGAGTCGGAGCGGCGAATGAGCAGTCGCAAAGCCGAAGCCGGGCGTGGCGCTTTAGAGCGTCCCAGGCTCTCCTTCGGAAAGATGTCGGACACGGCAGTAGCTGA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCAGCGGCCGCGGCGACAGCTCCAGCTCCGGCTCCGGCTCCGGCTCCGGCTCCGGCTCCCGCGCCTGCCCCGCTCGGCCCAGCGCGCCCGGGCTCCGCGCCCCGACCCCGCCGCCGCGCCTGCCGGGGGCCTCGGGCGCCCCCGCCGCCCGCCTCACGCTGAAGTTCCTGGCCGTGCTGCTGGCCGCGGGCATGCTGGCGTTCCTCGGTGCCGTCATCTGCATCATCGCCAGCGTGCCCCTGGCGGCCAGCCCGGCGCGGGCGCTGCCCGGCGGCGCCGACAATGCTTCGGTCGCCTCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CTTTTGAACATCTTGTGCCAGGATTTGACATTAAGCTCCAGTTCATCCGCACACCCCGCGGCCGCTTTCGCAAGGGACCGAGAGGGTGGCGGCCTCGCCCGGAGACTCGCGGCGCTGGGAGCTGCCTGCCCTCGGTGCGGGCTGCAGACCGGGCCGCGAGGCGGGCGCGCGGCGCGCACTAGGACCCAGCAGGGCTCCAGGCCGGGGTGGGAGCCGCGCCGCCGACCCCGGGCGGGCGCCGGGCGCAGGAGCCGGGGTTCCGGCCGCGATCTGCTGCAGCTCGGCCGGGAGACGGCGCGA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTGTGGCCAAAGAGAAGAAATGGGTTGAGACAGCAGGCCTGGCACTTACTTTACCTGGCCCAGTCTTGCCTGACAATTAAAAAAAGACGCTTTAGACTGGGCGCGGTGGCTCACGCCTGTAATCCTAGCACTTTGGGAGGCTGAGGCGGGCGGATCACGAGGCCAGGAGATTGAGACGATCCTGGCTAACACAGTGAAATCCCATCTCCACTAAAAATACAAAAACTTAGCCGGGCATGGTGGCGGGCGCCTGTAGTCCCAGCTACTCGGGCGGCTGAGGCAGGAGAATCGCTTGAACC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCATATAAACAGATTAACTGCATTCCCCAACGAGTAGAAATCTGCCCATCACTTTTGGGGTGGTTTGTTGCACCCACTATCGCCCAAGAAACTTCCCCAAGCCGGGTCATATTACAGGTCTCTAGCGCGCTGGCCGCACTCGCAGCCCACCATTCCCACAGAATGGTGTCATCGTAATCTGAAAAGTTGCCACTGGGCACCTGCGCAGGCTTGGCTGCGCCCTCTCGCGCCGCACGCTCCTTCCGGCGACGCGGATAGTCGTAAAGGTTTGAATACCTGATACAGGGAGAGAGGCCGAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCTTCACAGGGCACAGCCCCTGCAGAGCATCTTGGTCATTTGGAAGAGGACACGGTATCCCCTCTGGCCAGAGTATGTCAGAGAAGGAAGAGTAGGGCTTTTTTGTTTTGTTTTTTTTTAAAGGTGCTTGCTTGTTTAATGTAAATAATAGAAAGCCTTAATATCTTTTCTGTAACACGGAGTAATATTTTAATGTCATGTTTTGGATGTACATAATATATTTATAACAAAGCAGCAAGAGTCTACTTAACCTTGGCTGCCTCGTGGTGTTTCCTGGCTGGGTGGGGTGGGGGGTATCAA",
+        "output": "Non-promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGTATTCAGACCAGAACAAACTCGAGTGCCCCAGCTCCTCTGGCAATTAGGTAGCCAATAGTTGAGATTGCGTAGGCAACTACGTAAAATGCGCCGTGCCGTGCCCCTTTCTGCCACTCTGAGTTCGGTAACATAGCTCCGGTTGCTAGGTACTTAAGGTATCCAGTTCCCCCAGCGATTTGCATGAACCGAGAGGGAGTGTCTTCTGCCAACAGATCAGGCGGGGGATCATGAACTCATCCTCCCAGGGAATGCCGGTCGGGGATCCCCCGCGCAGCTCACAGGCCCTGGGAGTGAGCT",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACCTCGCTATTCTATGCACAGTTTTGAATCCTGCTTTTAAAATGCGATGTCTGCCTTGTCTTTAAATATTCTGAGGGGGAGGAGAGGAGATGTGTAATGCGACTGTCTATAATTTGAAACCGGACCTCACTATTTAGCGTCTCAGAAAACGCGGGGTTCATTGCCCTGGTCCCCCCCGCCTTCCCCCGCCCCCCGAACCTTCCGGCTGGCTCTCTTGCGGTCCCCGCCTCGGCGCTGATGTGGTCTGGCAGTGGAGATTGGCGCCCGGGCGGAGCACGATGGGCTTCCCCGCGGCAGGCA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCACGAGCGCCAGAGCCCCGCGCCTCCCCTAGCATATCCGGCGGAGTCCCTGCCCCGCAAGCGCCCACTATACGGCCGGACAACTGTTGTGGGCGTGGATGCAACCACGGGGGAGGGGGCAGGCGTGATCTCAGTCCCGCCCGTGCCCGAGCCCGCGCCCGAGCAGGGACTACATTTCCCTCCCCCCTGGGTCAGGGAGGAGTCTCCTCCTCTATCTATTAAGGATTCTCGCCTCCGCACATGCGTTATCCTTCCAAGGCCGAAAGGATTTGGAGTCCTGTACGCCAGAGGCCGAACTCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACCCTGCTGGGGGTACAGAGACGTTTCCGAAACTCAGCGGCTCGGTCGCCCCCAAAGGCCAGGCGGAACACACGCCCACACCGCGGCTCCCTTCCCGAAGTAAGACCGCCGGGCCACGGCCGCCCCCAGGAAGCCCCGCGCCCCGCACCCCACACCCGGGGACACAAACAGGCGCCGGGTAGCCGCGAGGGCCGGCGCGGCGCCTTTAAGAGGCGGCGGGCGGCGCTGCCCCCTGGCGGCCGCCCCGCCGCTTCCTCGCCGCCGCGGGCTCAAGCGGGGCGGCCGGGCCAGCGCGGGGCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AGCTTCACAGTCAGAACGCGGTACCGTTGTGCCCCAGCTCGACTGCAGCCTCCCCGCCCCGATACACTTGTCCCATCCTGTCCACGGCGCCCTTGTTTTGATTCGCTGCTGTCCCAGAGAGCTTAGTTATTCGTACGGGCAATCTTGACCCTTTCCCAGGACCTCCCCTAATACTGTTCATTGATCCTGGACCGGCAGCGAGACCCCTTCCGGCATCCAGAGGCCCAGTACTTTCAATCTCCTCCCCCAAAATGCTTTCCCTTTGTGGGGACCCTCGACATAGATGGACACCCAGCGGCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCGTTCTTGGCATCCTGAGAGGGCCAGTTAGTACTGAAGTCCCTTGGCTGCTCAAGGATTGCAGGGATGAGGCAAGTGGAACAGCCTCGGAACCTCCGAAAATGGGCACGCTCCAGGTCCCAGTTTCTATGGCAACCATACCGGCAAATTGGGCTCCGCAATGGTTTCTCCTGGAAAAACCGTGATTTTGGTTCCCGCGGACGTCTCTATGGTTTCGACAGCCTAGAAGGAACAAAACGGCATTTCCGGGAAGATGGCGGCGCACAAGTCAGGTCCGGCACATGTTTCCGCGGAGCGGAC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCTCTCGCGCCGCTCGCGTGACCGGCCGGTGTGTGCGCGAGGCCCCGGCTCCCGGGGCACGGACGGCCGGGCGCGCGCCTCTGCGAGGGGCGTCCGGGTCCGAGTCGGCGGTCCGGGCCGGCGCGAGGTGCGTGCGGGCGGGCCGCGGGGGTCCCGGACGGACACAAGCGCACACACTCCCGGAGGAGCCTTCGAGGCTGCTCTTCCTCGGCCAGACGGAGAGCGGCACTGTCTCCCCGCCCAGCGCTCACTCGCCCCGCGTCTCCCCCCGCGGCGGCTGCTCCTCCTCGGCACCGCCAG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GTCCCTCCTCTGGTAGCCAGAATAAATTTTGCAGTAAGTTAGCAGAGTTGATTCCAAAAAGTCTGCACTGTGGCATCTCAGGAAACCAAACTTATCTGCTTAAGAAATAACTTGCAGAGAGAAAAGGAACTCTGACAAGGCTAATAACAGGGATTTCAAGCTGTGTTTCAGCAAGCCAGTCACAGAGTATTCACGTGTTAATTCACTGGCGTGTTCCGCGGCATAAGCACCCCCTCTCTGCCTTCCCCCAGTTCCAACTGTTGTGCTGCAGCAGATTTGGTCTGAGTCTGGGCAGAGCCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ATGATGCTGGGGACGGGACACAAACAGGCGTGACAGCAGCGACCAGGCCGGCGCGGCGCCCGTAGACTGAGTCGAGACACCCTGGGCACAGGCGGCCGCCCCGCCTCTGTGATAGTTACTGGGACGTGTGCCCAACGGCCGGGCCAGCGCGGGGCGGCGGCGGGCAGGGGCGGCGAGTGCCTCGACGTTCTAGTGCGCCCTTCTCGGCGGTCTTATGCACTATGGCAGCAAGAGTTTTGAATATGTTGCGGTAAGTTGAAATTATCGAGAATGCCAGGCGAGTTGGCCGCGTGCGCCCCG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGCGACACGCCGTGCGCCTCCGCGGCTGCGCTACGAAAACGAGTCCCGGAGCGGCCCCGCGCCCGCCGCACCCGGCCCTCGCCCGCCCGAAGACAGGCGCCAAGCTGCCCCGCCGTCTCCCCAGCTAGCGCCCGGCCGCCGCCGCCTCGCGGGCCCCGGGCGGAAGGGGGCGGGGTCCCGATTCGCCCCGCCCCCGCGGAGGGATACGCGGCGCCGCGGCCCAAAACCCCCGGGCGAGGCGGCCGGGGCGGGTGAGGCGCTCCGCCTGCTGCGCGTCTACGCGGTCCCCGCGGGCCTTCC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTAGCCAGGAGGTAGCATCTGTCTCCAGGTGCCCGTGGGGTGTCCTCCATGAGGACCACCTCTTCTAACCACCTGCCCAGCTCAGAGGACCAGCTGGCAGGAAAGACGCTGCACCGCCCATGAGGCCAGAGCTACAAGTTGCTTCTAGGAGTGGCTGTGGGCGGAGGCTGGTGGTTCTGAAGGTGGCGGTGGTTCTGCAGCGTGGCTCCCCACAGCCCCTTTCATCTGAACAGTAGGGCTCATGCCACTTGTTAATCAGCCTTCCAGTGGCACGAGGAGTCATTTTCTGGGGATAGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCCTCCTCCCACGCTGCGCGCGCACCTCCCCGCCCCCACCCCTACCCGCTGGCGTGCCCAGTGGAACGGAGCCTTGTGTCTCCGCCTCAAGTCCCCGGATGCTCACCTCCCCGACTCGCCCCCGCTGTGGCCCCGCCCCCGCGCGGCTCTTCGTGCCACGTCACCGCCTGCGTCGCTTCCGGAGGCGCAGCGGGCGATGACGTAGAGGGACGTGCCCTCTATATGAGGTTGGGGAGCGGCTGAGTCGGCCTTTTCCGCCCGCTCCCCCCTCCCCCCGAGCGCCGCTCCGGCTGCACCGCG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCATGCAGCGACGGCCGCCGCGGAGCTCCGAGCAGCGGTAGCGCCCCCCTGTAAAGCGGTTCGCTATGCCGGGGCCACTGTGAACCCTGCCGCCTGCCGGAACACTCTTCGCTCCGGACCAGCTCAGCCTCTGATAAGCTGGACTCGGCACGCCCGCAACAAGCACCGAGGAGTTAAGAGAGCCGCAAGCGCAGGGAAGGCCTCCCCGCACGGGTGGGGGAAAGCGGCCGGTGCAGCGCGGGGACAGGCACTCGGGCTGGCACTGGCTGCTAGGGATGTCGTCCTGGATAAGGTGGCATG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCCCTGAGCCAGGCTGAAGCGGGATAGGAAGGGTAAAGGTGCTAGCTGAGCCGTAGTACGGCCCAGCAGTTCTCCAGGGGACAAGCTAGGCCCCCGCTGGAGGGAGTGGTGAGACTCAGCCGGGGCCTGATTCTGGTCAGCACCTTGGACAGCAGCCAGTGCCCTCCCTGGCCCCGCCTGGCCCCGCCTGGCCCCACATCCGCTACTCCTGGGCACCTCCTCAAATAGCACAGCCTCCAGCTGGTGCCCAGGGCCTGAATACACAGAGCGCTGAGAGAGTGGGGCAGTGTGGTCACGGAC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CCTGGGAAAAGCTGAGAGCATAATCGAGCGGCAAAGAAAACGCGACCACTGAGGACACTTAGAAGGGGGCGGGGCCTTTTTGTGGTACTAGGAGCTCGAGAAATTGACTAGATTACGAGTGGTTTACGACTCAAGCATACAACTTACAGCTTGGGCCTTCCGTAGCGGGCGTTGCGGGGGAACGTACAGTGCTTGAGGGGCGAGGTTTGCCGGAGCCTCATTCTAAAGGAATCTTCTAGTTCCAAGCGTTGTTCCAATTGGGCCGGAGCGTGCAACGGGACCCACGGAACTACAGGTGTG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AAAGCCACCTCCGCGCCGGCAGCTCCGTTTCTAGGGAAACAGGACCAATTCCAGAAGTTGGGGTGTTACGTGTGCGATATACCTGCCCAAGAGCTGGTATCAATAGAGCGGGCGGCATGGTAGTGTTTCCCCCGGCCATGGTAAAATGCGAACCAGATCCTCGGCGGGTAACCGCGCCCCTAGGCGTTGGCATGTGTCGGGGTAGCATGACGGGGTCGTCCAGTTCTTAGCCAAGATGAATCCGGCTCCGAAGGTCCCCGGCTTCGTGGCCATAAAGGCGAACACCGCTGGAAAGGTGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGGCTCGGGGACGGGACGGGCGAGCAGTCATTTCTGAATAGCGCGTGCAGAGGGTCATCGCGAAAAGACTAAGACGCGACATAATGGACTGACTCGGGGCATGGCGGGGTGGAGGCGGGAAAGTGTCGAGGCCGGCTAATGCCTGGACAGGGAGGTGGGCATCGCCAGGGCGACGATGAGGGTCGCCACTCCGGAACACGATTAGCATCAGTGCACTGACCTCTGCCCCCACGACTCTCCAACACGCCCGATGATGTTGCGTGCGGCAGCTGTCGGCTAGGAGTCGCGGGGTCTGTGCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATGGTATTTCAAGGTCATAAATTACAAATGTGCATATTTATTATTCCCCTATAAAATGTTAATTAAAAAATTTTCATATTTACTTGATGTAACAGAAATGAAACCTGTAATGAGGGATCTAATTTCTTTTTCTTTTTTCATTCCAGAATGTATGACAACATGTCCACAATGGTGTACATAAAGGAAGACAAGTTGGAGAAGCTTACACAGGATGAAATTATTTCTAAGACAAAGCAAGTAATTCAGGGGCTGGAAGCTTTGAAGAATGAGCACAATTCCATTTTACAAAGTTTGCTGGA",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "ACGACCCTGGCCTCCGACTTCAACGACTTCATAAGGCGGCGTTTCTGGGCGCAGCCGTGTCGCTCCTGGTGAGAGGCCGCCGGCAGGCGGGATCCAGCGCCCTCCGGGGCACCGCGGGCGAGACCGTCGCCTTCGCACCCCCGGCGCGGTCGACCCCGCGGGGCCGTCGGGTCCTGGGTTCCCCGCCGCGTTGCGCTCGTCCCCCTCCTGTCAGAACCTGGGCCCCCGCCCCGCCCACCGGCGCGGGGCCTCTCCTCCTCCCGCTAACGGGCGGTCGGCCGCCTCCTTCCTCTCTTGGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTTTCTGAGGCTGAGGAGGTACCCAGGTAAGCACACTCTGGGTTCTCTGGTGAGCACACACCGAGCTGCCTGCATCTCCCCTACCCATGAAGGCAGCTAGCCGCCGGGCCAGTGGTGGGAAGGGGAGGCGGCCATTAGCGCCGTTCCCGCCGGGACTTGAAGCGCCCGGCCGCGGCAAGCCCCGCCCTCGGCGCGCCCCCGCGTCCGCGCGCGCTCCTCGGGTCTGCGCGGAGCCGGCGTCGGCGCGCGCTTGGGCGCCTGGCGAAGACCGAGAGAGGCTGGCGGGATCTCAGCGGCGC",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCGCCCGTTGGACATACCTCTAAATTGGGAGCGCGGTGCCGATTTACGTGACCGCCGTGGTTCGCACGGTGGGTAAACAGTGGATCGTGGGGTGGTGAGAAGGTTCCGATGGGTGTTCTTACTACGGCCCGATCCTAGCAGTCGCCGACTTTGTTATTCTTATTGGTGGATGGGGGGCCGCTGATATAAATCTGGGAGATGAGGAGATAGGCGGGAGCGAGGGCTGACTCTGCGTGTCGAAGGAGGTAGAGAAAAAGCGTTGCTTAAAGGGAGGGAGCGTAGCTTGGTTGCTCCGTAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GACCTCTCTTCGGGCGGCAAGTGCCATACGTATAGATAAGCGGAATTTACATGGTGTCACAGTTAGCGCAAGGTCGAATAAATGCGGCTTAACACGACAGGTATCTCCAAATAAGTGTGTCTTACGGCGTAGATCGCGAGTTGAGCTGTCATTTCTTTCCAAGCGGCGCAGGACAGGACCCAGAAGCCAGAGCGCAGGAACTGCTTCCGGTTTGAGTTTTTATTAGAATAACGGGCTGTGGGAGAGGCAAGTGGCGGATCTTCAGGTCGGCGAGGGTGTGACATTCGACCCACAGGCCGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCATATTCTGTCTGAAATCGTGTGCACCGAAATCCCCGCCTTGCGGTGGAGGCTGGCGCTAGGCGGCCTCAGCCTCGGCCTGCTGCGCTCAGGAACCCGCGCCCCGGCTCCTCGGCGATCCATTGCTCTTTCCTCTGGCGCCGGCCGCAGGCCTCGGTCACGCCCCCAGCGGCCCGTTGGTTTCCGGGTCCCGCGGGGTGCCCCCGCCCACACGCTATGCCTTAAATTGGGCCAGGCTGAGGCGCTGCTGCTGGAGCGGCCGATCCGAGACGTGGCTCCCTGGGCGGCAGAACCATGTTG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TGGTCTTGTGTCTCGGCGTACCATATCTTTTGGGTTCCCCGGCTGCCCCGGAGGTTCGCGGCGAGAGGACGGGCCGGGTGCGGCTAGTTTACCTGAGCAGTGTGAACTACCTCACTCCCATCGGGACCAGCCGGCATAAACACTCTTATTGCCTGTCAACGTTTTCGAGACATATTTTGTTCGCGAGCCCGCAGACAACGCGCATTCTGGAGTTCGAACGCTCAGTTCGCGTCCCCGAGAAGATATTATTCAGATGCCTGCGAGCTGGGGCCGGGTGGCCGCGCCGGGTCGGAGTGTGGC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCCGGAGACAGGTGGGAGGGTGGGGGATGCCGGCAGGAGAGGGAAGCAGGACCGTGCGCGCCGCCACCCACCCGCAGTCCGTCCGCCCCCGGGCGCCCAGCGCGTGGCCCGAGCGGCGCTGCCCGCCGAGCCGGGAGCCCCCTCGGCCCCTCCCTTCAGGCGCGGGCCGGGGGCGGCTTCTCCGCGACCTTATGTAACCGGGCGGGAGGGGCCGGGCGGGCATGGGCCTTCCCGGCCCGGAGCTGGGAGTCGAAGGGGCGGGAGGCGTGATGGTGAACTCGCAAGAAGTTTGAGGGACG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GCAGGACTGAGTCACCTCCTGAAGGCCCCGTTCCAAAGCACTCCCACGGGGGATTAGGTTTCAACACAGGAATTTTTGGCGACACAAACATTGAGACCACAGCACCCAGCTCTCTGGGTTCCCAGAACCCTGTGGGAGGGCTGGGTCGCGCCTGCCCCCTTTTAAACACGAGGCTTCCATGGAAGCTTCTTGCCGGAGGCTGGACACTGAGGAATGGACAGGATCAACACTCTTCCACCGGCTCTTCCACCTCCCGATCACTTCCACCATCGCTGGGGGAAGAACGAATGCACCGCGTGG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CACAGCCCCAGATGGTGACTACTGGAGACTGCTGCCCCCAGGTATCCACATTGTCATTGCCCAAGCCCCTGGCTACGCCAAAGTCATCAAGAAAGTCATCATCCCCGCCCGGATGAAGAGGGCTGGCCGTGTGGACTTCATTCTGCAACCTCTGGGGATGGGACCCAAGAACTTTATTCATGGGCTGCGGAGGACTGGGCCCCACGACCCACTGGGAGGTGCCAGCTCTTTGGGGGAGGCCACGGAGCCCGACCCGCTCCGGGCGCGCAGGCAGCCCTCGGCCGACGGGAGTAAGCCCTG",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "AATCGTGTAATTCCCGCTTCCTATGGCAAGCGGCGCCCAGTTCTGGGTGACTTGTGAAATGAGCGTCCTGCTAGGCCATAGTCTCTACGAGGAGGATATAATGTAAAGTCTGGAGCGAACGTAAATGCCGTGACGTGTATTTGTGAGACTTCGCCCGGTCGGTGAGCTGGGTGCATAGGTTCCGAGCGCGCGACCACTTAGCTTCCCTTCTACAACAACCACCTGTGTGCGCATGCCCTCTCATCTTACAGTTGTCAAAGTGCTTTTGCCAGAGCGGTCACATAATCAACCTGAAAGAAG",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TCCTTAGAATATTGGGCATACAATTTACTTTTATAAATGTCTATGAAATAAAATTCTTATATAATTATATAAAATTACATAATATATAATTATATTGAAATAATTATAACTGTTGAATAAATGTATTATTTTTCCTTTGAACTATTTAACAGGGTTATATAACAAACTCATCACGAGTTGTGTCTAATAAGTCATCAGAGTTACTGTTTGACTTGACCCAGGATACAGGATTATCACATTACCAAGGGGGACCAACACTTTCTATGGCAGGTTGGTTTTAGTATTTTTTTCATAAAGGTT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "CGCGCGCGCGCCGGGCCGAGGGGCCGAGAGGCGGGGTCTTACAGCGACCGCGGGAAGAGGGCGCCCCAGAGCTGAGCCGGAGGCCGGCTGCCAGCGGGGCGAGGGTGGGCGCGAGCGCAGGGGCGGGCCCCGAGGGAGGGCGGGGACGGTGAGGGGGCGGGGTCGGGCTAGGCGGGGACGCGCTCGCGGGGTGGGGAAGGCGGGGGCGCGGCGGTGGCGGGAGCGTGCCCGGTCCCCGCCCCTGTTCCCACTCTCCTTCCACCTCGGACCGGCCGGGGCTCCGCAGAGCCAAAGCTCGCT",
+        "output": "promoter",
+        "model_response": "promoterpromoterpromo"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "TATTTCCGATTCCTTGACTAACGTACGGGGGCCCAGCACTTTCATTCCATCCCATCACCCCATGATCCTTCCCCAGACTCTAATACTGGCCGTAATATTTACCCCCCCCTGCTCCATGTGCGTAGATCAATGACACGAGCAAGCCATTGGTCTAAAACCCAGGGTTCTCTCGCAGAGGCCCTCCATTGTCTCTGCGGTGTCACTTCGCGCACACACATCAGAGCGGGCCTCTCTGTGAGGCGCGGGGTTGATGTACTCACACGCTGCGTTGATCGGAGGTCGCTGCCAAGCATGGCGCCC",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    },
+    {
+        "instruction": "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.",
+        "input": "GGTCAGCAAAGGCTTCCCTGAGGAATTGGGAATAGCCAGCCATACGGGTGGAGCCGGTCAACTCCTTGTGGTTGGGAGCTAACCGATAGCCCCTACAAGGAGCATTCCAACTATGTTATATAACATTGTGATTTTCGCGAAGGCAACGACTTATTCTTAAGACAGTATCTATCTGAATCTGAAGATCCTTGGTCTGGAGATTGGTAAAAATCTCCAGTCATCTTATGTTTCTATTATACTCTACTGGCGTTTTTACGTGACGCGGGGAGTCTACCTCCCCCAACCATAATTTAACAGTAA",
+        "output": "Non-promoter",
+        "model_response": "Non-promoter"
+    }
+]

04-gene-sft/gpt2_lora_text_classification/README.md

@@ -0,0 +1,202 @@
+---
+base_model: dnagpt/dna_gpt2_v0
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

04-gene-sft/gpt2_lora_text_classification/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "dnagpt/dna_gpt2_v0",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": true,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

04-gene-sft/gpt2_lora_text_classification/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:420bd2ac394927379af63039e7cf7cc8ce58744863a2ed076cc1710f3a5b22f1
+size 1188920

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/README.md

@@ -0,0 +1,202 @@
+---
+base_model: dnagpt/dna_gpt2_v0
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "dnagpt/dna_gpt2_v0",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": true,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:420bd2ac394927379af63039e7cf7cc8ce58744863a2ed076cc1710f3a5b22f1
+size 1188920

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/merges.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c79d89982c6ffe11f99a9830590377eba204aa277e9e00da4b44db9a758babd
+size 323115

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c652df18dc71913c4ff5b64303622dc2ff04d1b88200b58d9797fadf6661fc95
+size 2393043

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3d2234629fa3aa7c9b33952182cfbee58c837dbd0f86b57ab65e76090e887de
+size 14244

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef93b247d1663864b151670cf47ed8f850f23e8283b1785b667f0bb66522304c
+size 1064

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/trainer_state.json

@@ -0,0 +1,768 @@
+{
+  "best_metric": 0.91875,
+  "best_model_checkpoint": "./gpt2_lora_text_classification/checkpoint-46620",
+  "epoch": 7.0,
+  "eval_steps": 500,
+  "global_step": 46620,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07507507507507508,
+      "grad_norm": 0.6601091623306274,
+      "learning_rate": 1.9850750750750752e-05,
+      "loss": 0.2791,
+      "step": 500
+    },
+    {
+      "epoch": 0.15015015015015015,
+      "grad_norm": 15.015149116516113,
+      "learning_rate": 1.9700600600600603e-05,
+      "loss": 0.2868,
+      "step": 1000
+    },
+    {
+      "epoch": 0.22522522522522523,
+      "grad_norm": 6.76954984664917,
+      "learning_rate": 1.955045045045045e-05,
+      "loss": 0.2775,
+      "step": 1500
+    },
+    {
+      "epoch": 0.3003003003003003,
+      "grad_norm": 15.123693466186523,
+      "learning_rate": 1.94003003003003e-05,
+      "loss": 0.2902,
+      "step": 2000
+    },
+    {
+      "epoch": 0.37537537537537535,
+      "grad_norm": 14.731389045715332,
+      "learning_rate": 1.925015015015015e-05,
+      "loss": 0.2755,
+      "step": 2500
+    },
+    {
+      "epoch": 0.45045045045045046,
+      "grad_norm": 6.209009647369385,
+      "learning_rate": 1.91e-05,
+      "loss": 0.296,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5255255255255256,
+      "grad_norm": 36.11541748046875,
+      "learning_rate": 1.894984984984985e-05,
+      "loss": 0.2902,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6006006006006006,
+      "grad_norm": 13.450925827026367,
+      "learning_rate": 1.87996996996997e-05,
+      "loss": 0.26,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6756756756756757,
+      "grad_norm": 19.150455474853516,
+      "learning_rate": 1.864984984984985e-05,
+      "loss": 0.2725,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7507507507507507,
+      "grad_norm": 1.4110784530639648,
+      "learning_rate": 1.84996996996997e-05,
+      "loss": 0.2842,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8258258258258259,
+      "grad_norm": 1.0682216882705688,
+      "learning_rate": 1.834954954954955e-05,
+      "loss": 0.2743,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9009009009009009,
+      "grad_norm": 0.35149845480918884,
+      "learning_rate": 1.8199699699699703e-05,
+      "loss": 0.2616,
+      "step": 6000
+    },
+    {
+      "epoch": 0.975975975975976,
+      "grad_norm": 12.264653205871582,
+      "learning_rate": 1.8049549549549553e-05,
+      "loss": 0.2683,
+      "step": 6500
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9097972972972973,
+      "eval_f1": 0.9093378607809848,
+      "eval_loss": 0.3078426420688629,
+      "eval_precision": 0.9168093118794933,
+      "eval_recall": 0.901987201077804,
+      "eval_runtime": 6.9148,
+      "eval_samples_per_second": 856.136,
+      "eval_steps_per_second": 107.017,
+      "step": 6660
+    },
+    {
+      "epoch": 1.0510510510510511,
+      "grad_norm": 0.6039217114448547,
+      "learning_rate": 1.7899399399399403e-05,
+      "loss": 0.2609,
+      "step": 7000
+    },
+    {
+      "epoch": 1.1261261261261262,
+      "grad_norm": 15.304805755615234,
+      "learning_rate": 1.774924924924925e-05,
+      "loss": 0.2652,
+      "step": 7500
+    },
+    {
+      "epoch": 1.2012012012012012,
+      "grad_norm": 1.7410695552825928,
+      "learning_rate": 1.7599399399399403e-05,
+      "loss": 0.2701,
+      "step": 8000
+    },
+    {
+      "epoch": 1.2762762762762763,
+      "grad_norm": 0.29760709404945374,
+      "learning_rate": 1.7449249249249253e-05,
+      "loss": 0.2592,
+      "step": 8500
+    },
+    {
+      "epoch": 1.3513513513513513,
+      "grad_norm": 9.87925910949707,
+      "learning_rate": 1.72990990990991e-05,
+      "loss": 0.2571,
+      "step": 9000
+    },
+    {
+      "epoch": 1.4264264264264264,
+      "grad_norm": 3.006570339202881,
+      "learning_rate": 1.714894894894895e-05,
+      "loss": 0.2793,
+      "step": 9500
+    },
+    {
+      "epoch": 1.5015015015015014,
+      "grad_norm": 15.687701225280762,
+      "learning_rate": 1.69987987987988e-05,
+      "loss": 0.2658,
+      "step": 10000
+    },
+    {
+      "epoch": 1.5765765765765765,
+      "grad_norm": 14.420040130615234,
+      "learning_rate": 1.684894894894895e-05,
+      "loss": 0.2525,
+      "step": 10500
+    },
+    {
+      "epoch": 1.6516516516516515,
+      "grad_norm": 5.118183135986328,
+      "learning_rate": 1.66987987987988e-05,
+      "loss": 0.2773,
+      "step": 11000
+    },
+    {
+      "epoch": 1.7267267267267268,
+      "grad_norm": 4.803476333618164,
+      "learning_rate": 1.654864864864865e-05,
+      "loss": 0.2533,
+      "step": 11500
+    },
+    {
+      "epoch": 1.8018018018018018,
+      "grad_norm": 12.475725173950195,
+      "learning_rate": 1.63984984984985e-05,
+      "loss": 0.2778,
+      "step": 12000
+    },
+    {
+      "epoch": 1.8768768768768769,
+      "grad_norm": 0.4601811170578003,
+      "learning_rate": 1.624834834834835e-05,
+      "loss": 0.2595,
+      "step": 12500
+    },
+    {
+      "epoch": 1.951951951951952,
+      "grad_norm": 9.139686584472656,
+      "learning_rate": 1.60981981981982e-05,
+      "loss": 0.2874,
+      "step": 13000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9135135135135135,
+      "eval_f1": 0.9150912106135987,
+      "eval_loss": 0.27880367636680603,
+      "eval_precision": 0.9013394315583143,
+      "eval_recall": 0.9292691141798586,
+      "eval_runtime": 6.9239,
+      "eval_samples_per_second": 855.011,
+      "eval_steps_per_second": 106.876,
+      "step": 13320
+    },
+    {
+      "epoch": 2.027027027027027,
+      "grad_norm": 6.4723801612854,
+      "learning_rate": 1.594804804804805e-05,
+      "loss": 0.2424,
+      "step": 13500
+    },
+    {
+      "epoch": 2.1021021021021022,
+      "grad_norm": 2.6300477981567383,
+      "learning_rate": 1.57978978978979e-05,
+      "loss": 0.253,
+      "step": 14000
+    },
+    {
+      "epoch": 2.1771771771771773,
+      "grad_norm": 0.37813109159469604,
+      "learning_rate": 1.564804804804805e-05,
+      "loss": 0.2745,
+      "step": 14500
+    },
+    {
+      "epoch": 2.2522522522522523,
+      "grad_norm": 8.724522590637207,
+      "learning_rate": 1.54981981981982e-05,
+      "loss": 0.2575,
+      "step": 15000
+    },
+    {
+      "epoch": 2.3273273273273274,
+      "grad_norm": 16.1109619140625,
+      "learning_rate": 1.534804804804805e-05,
+      "loss": 0.2579,
+      "step": 15500
+    },
+    {
+      "epoch": 2.4024024024024024,
+      "grad_norm": 9.96057415008545,
+      "learning_rate": 1.51978978978979e-05,
+      "loss": 0.248,
+      "step": 16000
+    },
+    {
+      "epoch": 2.4774774774774775,
+      "grad_norm": 1.349557876586914,
+      "learning_rate": 1.504774774774775e-05,
+      "loss": 0.2422,
+      "step": 16500
+    },
+    {
+      "epoch": 2.5525525525525525,
+      "grad_norm": 0.23730000853538513,
+      "learning_rate": 1.48975975975976e-05,
+      "loss": 0.2905,
+      "step": 17000
+    },
+    {
+      "epoch": 2.6276276276276276,
+      "grad_norm": 13.847168922424316,
+      "learning_rate": 1.474774774774775e-05,
+      "loss": 0.2584,
+      "step": 17500
+    },
+    {
+      "epoch": 2.7027027027027026,
+      "grad_norm": 8.907866477966309,
+      "learning_rate": 1.45975975975976e-05,
+      "loss": 0.2855,
+      "step": 18000
+    },
+    {
+      "epoch": 2.7777777777777777,
+      "grad_norm": 9.752605438232422,
+      "learning_rate": 1.4447447447447448e-05,
+      "loss": 0.239,
+      "step": 18500
+    },
+    {
+      "epoch": 2.8528528528528527,
+      "grad_norm": 16.184972763061523,
+      "learning_rate": 1.4297297297297299e-05,
+      "loss": 0.2749,
+      "step": 19000
+    },
+    {
+      "epoch": 2.9279279279279278,
+      "grad_norm": 17.949094772338867,
+      "learning_rate": 1.4147147147147149e-05,
+      "loss": 0.2828,
+      "step": 19500
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.914527027027027,
+      "eval_f1": 0.9149579831932773,
+      "eval_loss": 0.29122158885002136,
+      "eval_precision": 0.9131164038913117,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.8853,
+      "eval_samples_per_second": 859.802,
+      "eval_steps_per_second": 107.475,
+      "step": 19980
+    },
+    {
+      "epoch": 3.003003003003003,
+      "grad_norm": 2.0944182872772217,
+      "learning_rate": 1.3996996996996999e-05,
+      "loss": 0.2426,
+      "step": 20000
+    },
+    {
+      "epoch": 3.078078078078078,
+      "grad_norm": 24.096668243408203,
+      "learning_rate": 1.3847147147147148e-05,
+      "loss": 0.2566,
+      "step": 20500
+    },
+    {
+      "epoch": 3.153153153153153,
+      "grad_norm": 22.673879623413086,
+      "learning_rate": 1.3696996996996998e-05,
+      "loss": 0.2579,
+      "step": 21000
+    },
+    {
+      "epoch": 3.2282282282282284,
+      "grad_norm": 14.733776092529297,
+      "learning_rate": 1.3546846846846849e-05,
+      "loss": 0.2613,
+      "step": 21500
+    },
+    {
+      "epoch": 3.3033033033033035,
+      "grad_norm": 3.1102752685546875,
+      "learning_rate": 1.3396696696696699e-05,
+      "loss": 0.2571,
+      "step": 22000
+    },
+    {
+      "epoch": 3.3783783783783785,
+      "grad_norm": 12.633967399597168,
+      "learning_rate": 1.3246546546546547e-05,
+      "loss": 0.2528,
+      "step": 22500
+    },
+    {
+      "epoch": 3.4534534534534536,
+      "grad_norm": 13.392036437988281,
+      "learning_rate": 1.3096696696696698e-05,
+      "loss": 0.2507,
+      "step": 23000
+    },
+    {
+      "epoch": 3.5285285285285286,
+      "grad_norm": 17.981300354003906,
+      "learning_rate": 1.2946546546546549e-05,
+      "loss": 0.2486,
+      "step": 23500
+    },
+    {
+      "epoch": 3.6036036036036037,
+      "grad_norm": 12.166417121887207,
+      "learning_rate": 1.2796396396396397e-05,
+      "loss": 0.2392,
+      "step": 24000
+    },
+    {
+      "epoch": 3.6786786786786787,
+      "grad_norm": 1.0378447771072388,
+      "learning_rate": 1.2646246246246247e-05,
+      "loss": 0.2799,
+      "step": 24500
+    },
+    {
+      "epoch": 3.7537537537537538,
+      "grad_norm": 18.767398834228516,
+      "learning_rate": 1.2496096096096097e-05,
+      "loss": 0.2654,
+      "step": 25000
+    },
+    {
+      "epoch": 3.828828828828829,
+      "grad_norm": 36.50725173950195,
+      "learning_rate": 1.2346246246246247e-05,
+      "loss": 0.245,
+      "step": 25500
+    },
+    {
+      "epoch": 3.903903903903904,
+      "grad_norm": 1.3716295957565308,
+      "learning_rate": 1.2196096096096097e-05,
+      "loss": 0.2468,
+      "step": 26000
+    },
+    {
+      "epoch": 3.978978978978979,
+      "grad_norm": 8.420230865478516,
+      "learning_rate": 1.2045945945945947e-05,
+      "loss": 0.2552,
+      "step": 26500
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.9163851351351351,
+      "eval_f1": 0.9187058630316965,
+      "eval_loss": 0.2815721333026886,
+      "eval_precision": 0.896474358974359,
+      "eval_recall": 0.9420680363758841,
+      "eval_runtime": 6.8819,
+      "eval_samples_per_second": 860.228,
+      "eval_steps_per_second": 107.529,
+      "step": 26640
+    },
+    {
+      "epoch": 4.054054054054054,
+      "grad_norm": 12.154544830322266,
+      "learning_rate": 1.1895795795795797e-05,
+      "loss": 0.2599,
+      "step": 27000
+    },
+    {
+      "epoch": 4.129129129129129,
+      "grad_norm": 18.989110946655273,
+      "learning_rate": 1.1745645645645648e-05,
+      "loss": 0.2299,
+      "step": 27500
+    },
+    {
+      "epoch": 4.2042042042042045,
+      "grad_norm": 15.256402969360352,
+      "learning_rate": 1.1595495495495496e-05,
+      "loss": 0.2793,
+      "step": 28000
+    },
+    {
+      "epoch": 4.2792792792792795,
+      "grad_norm": 0.3791729211807251,
+      "learning_rate": 1.1445645645645647e-05,
+      "loss": 0.2362,
+      "step": 28500
+    },
+    {
+      "epoch": 4.354354354354355,
+      "grad_norm": 0.3302192986011505,
+      "learning_rate": 1.1295495495495497e-05,
+      "loss": 0.2487,
+      "step": 29000
+    },
+    {
+      "epoch": 4.42942942942943,
+      "grad_norm": 0.19345639646053314,
+      "learning_rate": 1.1145345345345346e-05,
+      "loss": 0.2693,
+      "step": 29500
+    },
+    {
+      "epoch": 4.504504504504505,
+      "grad_norm": 0.5276665091514587,
+      "learning_rate": 1.0995195195195196e-05,
+      "loss": 0.2515,
+      "step": 30000
+    },
+    {
+      "epoch": 4.57957957957958,
+      "grad_norm": 17.693134307861328,
+      "learning_rate": 1.0845045045045046e-05,
+      "loss": 0.2398,
+      "step": 30500
+    },
+    {
+      "epoch": 4.654654654654655,
+      "grad_norm": 4.102345943450928,
+      "learning_rate": 1.0695195195195196e-05,
+      "loss": 0.2534,
+      "step": 31000
+    },
+    {
+      "epoch": 4.72972972972973,
+      "grad_norm": 14.490029335021973,
+      "learning_rate": 1.0545045045045046e-05,
+      "loss": 0.2591,
+      "step": 31500
+    },
+    {
+      "epoch": 4.804804804804805,
+      "grad_norm": 0.18800705671310425,
+      "learning_rate": 1.0394894894894896e-05,
+      "loss": 0.2433,
+      "step": 32000
+    },
+    {
+      "epoch": 4.87987987987988,
+      "grad_norm": 1.4547662734985352,
+      "learning_rate": 1.0244744744744746e-05,
+      "loss": 0.2514,
+      "step": 32500
+    },
+    {
+      "epoch": 4.954954954954955,
+      "grad_norm": 23.680500030517578,
+      "learning_rate": 1.0094894894894895e-05,
+      "loss": 0.252,
+      "step": 33000
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.914527027027027,
+      "eval_f1": 0.9149579831932773,
+      "eval_loss": 0.27195000648498535,
+      "eval_precision": 0.9131164038913117,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.9368,
+      "eval_samples_per_second": 853.416,
+      "eval_steps_per_second": 106.677,
+      "step": 33300
+    },
+    {
+      "epoch": 5.03003003003003,
+      "grad_norm": 0.7097603678703308,
+      "learning_rate": 9.944744744744746e-06,
+      "loss": 0.2542,
+      "step": 33500
+    },
+    {
+      "epoch": 5.105105105105105,
+      "grad_norm": 9.578977584838867,
+      "learning_rate": 9.794594594594596e-06,
+      "loss": 0.2614,
+      "step": 34000
+    },
+    {
+      "epoch": 5.18018018018018,
+      "grad_norm": 55.42725372314453,
+      "learning_rate": 9.644444444444444e-06,
+      "loss": 0.2489,
+      "step": 34500
+    },
+    {
+      "epoch": 5.255255255255255,
+      "grad_norm": 9.72994613647461,
+      "learning_rate": 9.494594594594595e-06,
+      "loss": 0.2535,
+      "step": 35000
+    },
+    {
+      "epoch": 5.33033033033033,
+      "grad_norm": 9.581122398376465,
+      "learning_rate": 9.344444444444446e-06,
+      "loss": 0.2594,
+      "step": 35500
+    },
+    {
+      "epoch": 5.405405405405405,
+      "grad_norm": 8.783069610595703,
+      "learning_rate": 9.194294294294294e-06,
+      "loss": 0.2499,
+      "step": 36000
+    },
+    {
+      "epoch": 5.48048048048048,
+      "grad_norm": 21.566892623901367,
+      "learning_rate": 9.044144144144144e-06,
+      "loss": 0.2391,
+      "step": 36500
+    },
+    {
+      "epoch": 5.555555555555555,
+      "grad_norm": 0.5095661878585815,
+      "learning_rate": 8.893993993993994e-06,
+      "loss": 0.228,
+      "step": 37000
+    },
+    {
+      "epoch": 5.63063063063063,
+      "grad_norm": 0.5652422904968262,
+      "learning_rate": 8.743843843843845e-06,
+      "loss": 0.2354,
+      "step": 37500
+    },
+    {
+      "epoch": 5.7057057057057055,
+      "grad_norm": 22.595531463623047,
+      "learning_rate": 8.593693693693695e-06,
+      "loss": 0.2313,
+      "step": 38000
+    },
+    {
+      "epoch": 5.7807807807807805,
+      "grad_norm": 10.465389251708984,
+      "learning_rate": 8.443543543543543e-06,
+      "loss": 0.2495,
+      "step": 38500
+    },
+    {
+      "epoch": 5.8558558558558556,
+      "grad_norm": 13.11806583404541,
+      "learning_rate": 8.293693693693694e-06,
+      "loss": 0.2603,
+      "step": 39000
+    },
+    {
+      "epoch": 5.930930930930931,
+      "grad_norm": 13.471338272094727,
+      "learning_rate": 8.143543543543545e-06,
+      "loss": 0.2423,
+      "step": 39500
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.9163851351351351,
+      "eval_f1": 0.9166526351237582,
+      "eval_loss": 0.28819864988327026,
+      "eval_precision": 0.9164983164983165,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.9985,
+      "eval_samples_per_second": 845.898,
+      "eval_steps_per_second": 105.737,
+      "step": 39960
+    },
+    {
+      "epoch": 6.006006006006006,
+      "grad_norm": 1.7739392518997192,
+      "learning_rate": 7.993393393393393e-06,
+      "loss": 0.247,
+      "step": 40000
+    },
+    {
+      "epoch": 6.081081081081081,
+      "grad_norm": 7.755850315093994,
+      "learning_rate": 7.843243243243243e-06,
+      "loss": 0.2288,
+      "step": 40500
+    },
+    {
+      "epoch": 6.156156156156156,
+      "grad_norm": 4.462266445159912,
+      "learning_rate": 7.693093093093093e-06,
+      "loss": 0.2566,
+      "step": 41000
+    },
+    {
+      "epoch": 6.231231231231231,
+      "grad_norm": 14.290399551391602,
+      "learning_rate": 7.543243243243244e-06,
+      "loss": 0.2559,
+      "step": 41500
+    },
+    {
+      "epoch": 6.306306306306306,
+      "grad_norm": 14.638737678527832,
+      "learning_rate": 7.393093093093093e-06,
+      "loss": 0.2237,
+      "step": 42000
+    },
+    {
+      "epoch": 6.381381381381382,
+      "grad_norm": 0.5656157732009888,
+      "learning_rate": 7.242942942942943e-06,
+      "loss": 0.2517,
+      "step": 42500
+    },
+    {
+      "epoch": 6.456456456456457,
+      "grad_norm": 5.043838977813721,
+      "learning_rate": 7.092792792792793e-06,
+      "loss": 0.2511,
+      "step": 43000
+    },
+    {
+      "epoch": 6.531531531531532,
+      "grad_norm": 1.055748462677002,
+      "learning_rate": 6.942942942942944e-06,
+      "loss": 0.2385,
+      "step": 43500
+    },
+    {
+      "epoch": 6.606606606606607,
+      "grad_norm": 0.2881987392902374,
+      "learning_rate": 6.792792792792793e-06,
+      "loss": 0.2537,
+      "step": 44000
+    },
+    {
+      "epoch": 6.681681681681682,
+      "grad_norm": 26.814804077148438,
+      "learning_rate": 6.642642642642643e-06,
+      "loss": 0.2544,
+      "step": 44500
+    },
+    {
+      "epoch": 6.756756756756757,
+      "grad_norm": 3.904442548751831,
+      "learning_rate": 6.4924924924924924e-06,
+      "loss": 0.2322,
+      "step": 45000
+    },
+    {
+      "epoch": 6.831831831831832,
+      "grad_norm": 0.3679012358188629,
+      "learning_rate": 6.342642642642643e-06,
+      "loss": 0.2314,
+      "step": 45500
+    },
+    {
+      "epoch": 6.906906906906907,
+      "grad_norm": 0.13983987271785736,
+      "learning_rate": 6.192492492492494e-06,
+      "loss": 0.2559,
+      "step": 46000
+    },
+    {
+      "epoch": 6.981981981981982,
+      "grad_norm": 13.696492195129395,
+      "learning_rate": 6.042342342342343e-06,
+      "loss": 0.2535,
+      "step": 46500
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.91875,
+      "eval_f1": 0.9199267521225237,
+      "eval_loss": 0.2686730921268463,
+      "eval_precision": 0.9094799210006583,
+      "eval_recall": 0.9306163691478613,
+      "eval_runtime": 6.8805,
+      "eval_samples_per_second": 860.409,
+      "eval_steps_per_second": 107.551,
+      "step": 46620
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 66600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.44459272830976e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-46620/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e641b388947ac47c75e10e6525af9bbc8a067f3617ad53397af08cf2c7e505
+size 5304

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/README.md

@@ -0,0 +1,202 @@
+---
+base_model: dnagpt/dna_gpt2_v0
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "dnagpt/dna_gpt2_v0",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": true,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8735630e2a1c93b91017c41487698662f8d4e7944955bcdb26edd08027cbaa60
+size 1188920

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/merges.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c79d89982c6ffe11f99a9830590377eba204aa277e9e00da4b44db9a758babd
+size 323115

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be30cc6993dc0245ec03aabb8f719b5ce79b7444959f2f45205f83b3e10921da
+size 2393043

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c78e95d7f2ce329e1ecbb8ee93a141949f25fff087fd73bd658b885c2b131d4
+size 14244

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0144db6a26dfbd90570a8a7979f3a1e463104b56b08635ada71ef34fdace1e16
+size 1064

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/trainer_state.json

@@ -0,0 +1,1084 @@
+{
+  "best_metric": 0.91875,
+  "best_model_checkpoint": "./gpt2_lora_text_classification/checkpoint-46620",
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 66600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07507507507507508,
+      "grad_norm": 0.6601091623306274,
+      "learning_rate": 1.9850750750750752e-05,
+      "loss": 0.2791,
+      "step": 500
+    },
+    {
+      "epoch": 0.15015015015015015,
+      "grad_norm": 15.015149116516113,
+      "learning_rate": 1.9700600600600603e-05,
+      "loss": 0.2868,
+      "step": 1000
+    },
+    {
+      "epoch": 0.22522522522522523,
+      "grad_norm": 6.76954984664917,
+      "learning_rate": 1.955045045045045e-05,
+      "loss": 0.2775,
+      "step": 1500
+    },
+    {
+      "epoch": 0.3003003003003003,
+      "grad_norm": 15.123693466186523,
+      "learning_rate": 1.94003003003003e-05,
+      "loss": 0.2902,
+      "step": 2000
+    },
+    {
+      "epoch": 0.37537537537537535,
+      "grad_norm": 14.731389045715332,
+      "learning_rate": 1.925015015015015e-05,
+      "loss": 0.2755,
+      "step": 2500
+    },
+    {
+      "epoch": 0.45045045045045046,
+      "grad_norm": 6.209009647369385,
+      "learning_rate": 1.91e-05,
+      "loss": 0.296,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5255255255255256,
+      "grad_norm": 36.11541748046875,
+      "learning_rate": 1.894984984984985e-05,
+      "loss": 0.2902,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6006006006006006,
+      "grad_norm": 13.450925827026367,
+      "learning_rate": 1.87996996996997e-05,
+      "loss": 0.26,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6756756756756757,
+      "grad_norm": 19.150455474853516,
+      "learning_rate": 1.864984984984985e-05,
+      "loss": 0.2725,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7507507507507507,
+      "grad_norm": 1.4110784530639648,
+      "learning_rate": 1.84996996996997e-05,
+      "loss": 0.2842,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8258258258258259,
+      "grad_norm": 1.0682216882705688,
+      "learning_rate": 1.834954954954955e-05,
+      "loss": 0.2743,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9009009009009009,
+      "grad_norm": 0.35149845480918884,
+      "learning_rate": 1.8199699699699703e-05,
+      "loss": 0.2616,
+      "step": 6000
+    },
+    {
+      "epoch": 0.975975975975976,
+      "grad_norm": 12.264653205871582,
+      "learning_rate": 1.8049549549549553e-05,
+      "loss": 0.2683,
+      "step": 6500
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9097972972972973,
+      "eval_f1": 0.9093378607809848,
+      "eval_loss": 0.3078426420688629,
+      "eval_precision": 0.9168093118794933,
+      "eval_recall": 0.901987201077804,
+      "eval_runtime": 6.9148,
+      "eval_samples_per_second": 856.136,
+      "eval_steps_per_second": 107.017,
+      "step": 6660
+    },
+    {
+      "epoch": 1.0510510510510511,
+      "grad_norm": 0.6039217114448547,
+      "learning_rate": 1.7899399399399403e-05,
+      "loss": 0.2609,
+      "step": 7000
+    },
+    {
+      "epoch": 1.1261261261261262,
+      "grad_norm": 15.304805755615234,
+      "learning_rate": 1.774924924924925e-05,
+      "loss": 0.2652,
+      "step": 7500
+    },
+    {
+      "epoch": 1.2012012012012012,
+      "grad_norm": 1.7410695552825928,
+      "learning_rate": 1.7599399399399403e-05,
+      "loss": 0.2701,
+      "step": 8000
+    },
+    {
+      "epoch": 1.2762762762762763,
+      "grad_norm": 0.29760709404945374,
+      "learning_rate": 1.7449249249249253e-05,
+      "loss": 0.2592,
+      "step": 8500
+    },
+    {
+      "epoch": 1.3513513513513513,
+      "grad_norm": 9.87925910949707,
+      "learning_rate": 1.72990990990991e-05,
+      "loss": 0.2571,
+      "step": 9000
+    },
+    {
+      "epoch": 1.4264264264264264,
+      "grad_norm": 3.006570339202881,
+      "learning_rate": 1.714894894894895e-05,
+      "loss": 0.2793,
+      "step": 9500
+    },
+    {
+      "epoch": 1.5015015015015014,
+      "grad_norm": 15.687701225280762,
+      "learning_rate": 1.69987987987988e-05,
+      "loss": 0.2658,
+      "step": 10000
+    },
+    {
+      "epoch": 1.5765765765765765,
+      "grad_norm": 14.420040130615234,
+      "learning_rate": 1.684894894894895e-05,
+      "loss": 0.2525,
+      "step": 10500
+    },
+    {
+      "epoch": 1.6516516516516515,
+      "grad_norm": 5.118183135986328,
+      "learning_rate": 1.66987987987988e-05,
+      "loss": 0.2773,
+      "step": 11000
+    },
+    {
+      "epoch": 1.7267267267267268,
+      "grad_norm": 4.803476333618164,
+      "learning_rate": 1.654864864864865e-05,
+      "loss": 0.2533,
+      "step": 11500
+    },
+    {
+      "epoch": 1.8018018018018018,
+      "grad_norm": 12.475725173950195,
+      "learning_rate": 1.63984984984985e-05,
+      "loss": 0.2778,
+      "step": 12000
+    },
+    {
+      "epoch": 1.8768768768768769,
+      "grad_norm": 0.4601811170578003,
+      "learning_rate": 1.624834834834835e-05,
+      "loss": 0.2595,
+      "step": 12500
+    },
+    {
+      "epoch": 1.951951951951952,
+      "grad_norm": 9.139686584472656,
+      "learning_rate": 1.60981981981982e-05,
+      "loss": 0.2874,
+      "step": 13000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9135135135135135,
+      "eval_f1": 0.9150912106135987,
+      "eval_loss": 0.27880367636680603,
+      "eval_precision": 0.9013394315583143,
+      "eval_recall": 0.9292691141798586,
+      "eval_runtime": 6.9239,
+      "eval_samples_per_second": 855.011,
+      "eval_steps_per_second": 106.876,
+      "step": 13320
+    },
+    {
+      "epoch": 2.027027027027027,
+      "grad_norm": 6.4723801612854,
+      "learning_rate": 1.594804804804805e-05,
+      "loss": 0.2424,
+      "step": 13500
+    },
+    {
+      "epoch": 2.1021021021021022,
+      "grad_norm": 2.6300477981567383,
+      "learning_rate": 1.57978978978979e-05,
+      "loss": 0.253,
+      "step": 14000
+    },
+    {
+      "epoch": 2.1771771771771773,
+      "grad_norm": 0.37813109159469604,
+      "learning_rate": 1.564804804804805e-05,
+      "loss": 0.2745,
+      "step": 14500
+    },
+    {
+      "epoch": 2.2522522522522523,
+      "grad_norm": 8.724522590637207,
+      "learning_rate": 1.54981981981982e-05,
+      "loss": 0.2575,
+      "step": 15000
+    },
+    {
+      "epoch": 2.3273273273273274,
+      "grad_norm": 16.1109619140625,
+      "learning_rate": 1.534804804804805e-05,
+      "loss": 0.2579,
+      "step": 15500
+    },
+    {
+      "epoch": 2.4024024024024024,
+      "grad_norm": 9.96057415008545,
+      "learning_rate": 1.51978978978979e-05,
+      "loss": 0.248,
+      "step": 16000
+    },
+    {
+      "epoch": 2.4774774774774775,
+      "grad_norm": 1.349557876586914,
+      "learning_rate": 1.504774774774775e-05,
+      "loss": 0.2422,
+      "step": 16500
+    },
+    {
+      "epoch": 2.5525525525525525,
+      "grad_norm": 0.23730000853538513,
+      "learning_rate": 1.48975975975976e-05,
+      "loss": 0.2905,
+      "step": 17000
+    },
+    {
+      "epoch": 2.6276276276276276,
+      "grad_norm": 13.847168922424316,
+      "learning_rate": 1.474774774774775e-05,
+      "loss": 0.2584,
+      "step": 17500
+    },
+    {
+      "epoch": 2.7027027027027026,
+      "grad_norm": 8.907866477966309,
+      "learning_rate": 1.45975975975976e-05,
+      "loss": 0.2855,
+      "step": 18000
+    },
+    {
+      "epoch": 2.7777777777777777,
+      "grad_norm": 9.752605438232422,
+      "learning_rate": 1.4447447447447448e-05,
+      "loss": 0.239,
+      "step": 18500
+    },
+    {
+      "epoch": 2.8528528528528527,
+      "grad_norm": 16.184972763061523,
+      "learning_rate": 1.4297297297297299e-05,
+      "loss": 0.2749,
+      "step": 19000
+    },
+    {
+      "epoch": 2.9279279279279278,
+      "grad_norm": 17.949094772338867,
+      "learning_rate": 1.4147147147147149e-05,
+      "loss": 0.2828,
+      "step": 19500
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.914527027027027,
+      "eval_f1": 0.9149579831932773,
+      "eval_loss": 0.29122158885002136,
+      "eval_precision": 0.9131164038913117,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.8853,
+      "eval_samples_per_second": 859.802,
+      "eval_steps_per_second": 107.475,
+      "step": 19980
+    },
+    {
+      "epoch": 3.003003003003003,
+      "grad_norm": 2.0944182872772217,
+      "learning_rate": 1.3996996996996999e-05,
+      "loss": 0.2426,
+      "step": 20000
+    },
+    {
+      "epoch": 3.078078078078078,
+      "grad_norm": 24.096668243408203,
+      "learning_rate": 1.3847147147147148e-05,
+      "loss": 0.2566,
+      "step": 20500
+    },
+    {
+      "epoch": 3.153153153153153,
+      "grad_norm": 22.673879623413086,
+      "learning_rate": 1.3696996996996998e-05,
+      "loss": 0.2579,
+      "step": 21000
+    },
+    {
+      "epoch": 3.2282282282282284,
+      "grad_norm": 14.733776092529297,
+      "learning_rate": 1.3546846846846849e-05,
+      "loss": 0.2613,
+      "step": 21500
+    },
+    {
+      "epoch": 3.3033033033033035,
+      "grad_norm": 3.1102752685546875,
+      "learning_rate": 1.3396696696696699e-05,
+      "loss": 0.2571,
+      "step": 22000
+    },
+    {
+      "epoch": 3.3783783783783785,
+      "grad_norm": 12.633967399597168,
+      "learning_rate": 1.3246546546546547e-05,
+      "loss": 0.2528,
+      "step": 22500
+    },
+    {
+      "epoch": 3.4534534534534536,
+      "grad_norm": 13.392036437988281,
+      "learning_rate": 1.3096696696696698e-05,
+      "loss": 0.2507,
+      "step": 23000
+    },
+    {
+      "epoch": 3.5285285285285286,
+      "grad_norm": 17.981300354003906,
+      "learning_rate": 1.2946546546546549e-05,
+      "loss": 0.2486,
+      "step": 23500
+    },
+    {
+      "epoch": 3.6036036036036037,
+      "grad_norm": 12.166417121887207,
+      "learning_rate": 1.2796396396396397e-05,
+      "loss": 0.2392,
+      "step": 24000
+    },
+    {
+      "epoch": 3.6786786786786787,
+      "grad_norm": 1.0378447771072388,
+      "learning_rate": 1.2646246246246247e-05,
+      "loss": 0.2799,
+      "step": 24500
+    },
+    {
+      "epoch": 3.7537537537537538,
+      "grad_norm": 18.767398834228516,
+      "learning_rate": 1.2496096096096097e-05,
+      "loss": 0.2654,
+      "step": 25000
+    },
+    {
+      "epoch": 3.828828828828829,
+      "grad_norm": 36.50725173950195,
+      "learning_rate": 1.2346246246246247e-05,
+      "loss": 0.245,
+      "step": 25500
+    },
+    {
+      "epoch": 3.903903903903904,
+      "grad_norm": 1.3716295957565308,
+      "learning_rate": 1.2196096096096097e-05,
+      "loss": 0.2468,
+      "step": 26000
+    },
+    {
+      "epoch": 3.978978978978979,
+      "grad_norm": 8.420230865478516,
+      "learning_rate": 1.2045945945945947e-05,
+      "loss": 0.2552,
+      "step": 26500
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.9163851351351351,
+      "eval_f1": 0.9187058630316965,
+      "eval_loss": 0.2815721333026886,
+      "eval_precision": 0.896474358974359,
+      "eval_recall": 0.9420680363758841,
+      "eval_runtime": 6.8819,
+      "eval_samples_per_second": 860.228,
+      "eval_steps_per_second": 107.529,
+      "step": 26640
+    },
+    {
+      "epoch": 4.054054054054054,
+      "grad_norm": 12.154544830322266,
+      "learning_rate": 1.1895795795795797e-05,
+      "loss": 0.2599,
+      "step": 27000
+    },
+    {
+      "epoch": 4.129129129129129,
+      "grad_norm": 18.989110946655273,
+      "learning_rate": 1.1745645645645648e-05,
+      "loss": 0.2299,
+      "step": 27500
+    },
+    {
+      "epoch": 4.2042042042042045,
+      "grad_norm": 15.256402969360352,
+      "learning_rate": 1.1595495495495496e-05,
+      "loss": 0.2793,
+      "step": 28000
+    },
+    {
+      "epoch": 4.2792792792792795,
+      "grad_norm": 0.3791729211807251,
+      "learning_rate": 1.1445645645645647e-05,
+      "loss": 0.2362,
+      "step": 28500
+    },
+    {
+      "epoch": 4.354354354354355,
+      "grad_norm": 0.3302192986011505,
+      "learning_rate": 1.1295495495495497e-05,
+      "loss": 0.2487,
+      "step": 29000
+    },
+    {
+      "epoch": 4.42942942942943,
+      "grad_norm": 0.19345639646053314,
+      "learning_rate": 1.1145345345345346e-05,
+      "loss": 0.2693,
+      "step": 29500
+    },
+    {
+      "epoch": 4.504504504504505,
+      "grad_norm": 0.5276665091514587,
+      "learning_rate": 1.0995195195195196e-05,
+      "loss": 0.2515,
+      "step": 30000
+    },
+    {
+      "epoch": 4.57957957957958,
+      "grad_norm": 17.693134307861328,
+      "learning_rate": 1.0845045045045046e-05,
+      "loss": 0.2398,
+      "step": 30500
+    },
+    {
+      "epoch": 4.654654654654655,
+      "grad_norm": 4.102345943450928,
+      "learning_rate": 1.0695195195195196e-05,
+      "loss": 0.2534,
+      "step": 31000
+    },
+    {
+      "epoch": 4.72972972972973,
+      "grad_norm": 14.490029335021973,
+      "learning_rate": 1.0545045045045046e-05,
+      "loss": 0.2591,
+      "step": 31500
+    },
+    {
+      "epoch": 4.804804804804805,
+      "grad_norm": 0.18800705671310425,
+      "learning_rate": 1.0394894894894896e-05,
+      "loss": 0.2433,
+      "step": 32000
+    },
+    {
+      "epoch": 4.87987987987988,
+      "grad_norm": 1.4547662734985352,
+      "learning_rate": 1.0244744744744746e-05,
+      "loss": 0.2514,
+      "step": 32500
+    },
+    {
+      "epoch": 4.954954954954955,
+      "grad_norm": 23.680500030517578,
+      "learning_rate": 1.0094894894894895e-05,
+      "loss": 0.252,
+      "step": 33000
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.914527027027027,
+      "eval_f1": 0.9149579831932773,
+      "eval_loss": 0.27195000648498535,
+      "eval_precision": 0.9131164038913117,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.9368,
+      "eval_samples_per_second": 853.416,
+      "eval_steps_per_second": 106.677,
+      "step": 33300
+    },
+    {
+      "epoch": 5.03003003003003,
+      "grad_norm": 0.7097603678703308,
+      "learning_rate": 9.944744744744746e-06,
+      "loss": 0.2542,
+      "step": 33500
+    },
+    {
+      "epoch": 5.105105105105105,
+      "grad_norm": 9.578977584838867,
+      "learning_rate": 9.794594594594596e-06,
+      "loss": 0.2614,
+      "step": 34000
+    },
+    {
+      "epoch": 5.18018018018018,
+      "grad_norm": 55.42725372314453,
+      "learning_rate": 9.644444444444444e-06,
+      "loss": 0.2489,
+      "step": 34500
+    },
+    {
+      "epoch": 5.255255255255255,
+      "grad_norm": 9.72994613647461,
+      "learning_rate": 9.494594594594595e-06,
+      "loss": 0.2535,
+      "step": 35000
+    },
+    {
+      "epoch": 5.33033033033033,
+      "grad_norm": 9.581122398376465,
+      "learning_rate": 9.344444444444446e-06,
+      "loss": 0.2594,
+      "step": 35500
+    },
+    {
+      "epoch": 5.405405405405405,
+      "grad_norm": 8.783069610595703,
+      "learning_rate": 9.194294294294294e-06,
+      "loss": 0.2499,
+      "step": 36000
+    },
+    {
+      "epoch": 5.48048048048048,
+      "grad_norm": 21.566892623901367,
+      "learning_rate": 9.044144144144144e-06,
+      "loss": 0.2391,
+      "step": 36500
+    },
+    {
+      "epoch": 5.555555555555555,
+      "grad_norm": 0.5095661878585815,
+      "learning_rate": 8.893993993993994e-06,
+      "loss": 0.228,
+      "step": 37000
+    },
+    {
+      "epoch": 5.63063063063063,
+      "grad_norm": 0.5652422904968262,
+      "learning_rate": 8.743843843843845e-06,
+      "loss": 0.2354,
+      "step": 37500
+    },
+    {
+      "epoch": 5.7057057057057055,
+      "grad_norm": 22.595531463623047,
+      "learning_rate": 8.593693693693695e-06,
+      "loss": 0.2313,
+      "step": 38000
+    },
+    {
+      "epoch": 5.7807807807807805,
+      "grad_norm": 10.465389251708984,
+      "learning_rate": 8.443543543543543e-06,
+      "loss": 0.2495,
+      "step": 38500
+    },
+    {
+      "epoch": 5.8558558558558556,
+      "grad_norm": 13.11806583404541,
+      "learning_rate": 8.293693693693694e-06,
+      "loss": 0.2603,
+      "step": 39000
+    },
+    {
+      "epoch": 5.930930930930931,
+      "grad_norm": 13.471338272094727,
+      "learning_rate": 8.143543543543545e-06,
+      "loss": 0.2423,
+      "step": 39500
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.9163851351351351,
+      "eval_f1": 0.9166526351237582,
+      "eval_loss": 0.28819864988327026,
+      "eval_precision": 0.9164983164983165,
+      "eval_recall": 0.9168070057258336,
+      "eval_runtime": 6.9985,
+      "eval_samples_per_second": 845.898,
+      "eval_steps_per_second": 105.737,
+      "step": 39960
+    },
+    {
+      "epoch": 6.006006006006006,
+      "grad_norm": 1.7739392518997192,
+      "learning_rate": 7.993393393393393e-06,
+      "loss": 0.247,
+      "step": 40000
+    },
+    {
+      "epoch": 6.081081081081081,
+      "grad_norm": 7.755850315093994,
+      "learning_rate": 7.843243243243243e-06,
+      "loss": 0.2288,
+      "step": 40500
+    },
+    {
+      "epoch": 6.156156156156156,
+      "grad_norm": 4.462266445159912,
+      "learning_rate": 7.693093093093093e-06,
+      "loss": 0.2566,
+      "step": 41000
+    },
+    {
+      "epoch": 6.231231231231231,
+      "grad_norm": 14.290399551391602,
+      "learning_rate": 7.543243243243244e-06,
+      "loss": 0.2559,
+      "step": 41500
+    },
+    {
+      "epoch": 6.306306306306306,
+      "grad_norm": 14.638737678527832,
+      "learning_rate": 7.393093093093093e-06,
+      "loss": 0.2237,
+      "step": 42000
+    },
+    {
+      "epoch": 6.381381381381382,
+      "grad_norm": 0.5656157732009888,
+      "learning_rate": 7.242942942942943e-06,
+      "loss": 0.2517,
+      "step": 42500
+    },
+    {
+      "epoch": 6.456456456456457,
+      "grad_norm": 5.043838977813721,
+      "learning_rate": 7.092792792792793e-06,
+      "loss": 0.2511,
+      "step": 43000
+    },
+    {
+      "epoch": 6.531531531531532,
+      "grad_norm": 1.055748462677002,
+      "learning_rate": 6.942942942942944e-06,
+      "loss": 0.2385,
+      "step": 43500
+    },
+    {
+      "epoch": 6.606606606606607,
+      "grad_norm": 0.2881987392902374,
+      "learning_rate": 6.792792792792793e-06,
+      "loss": 0.2537,
+      "step": 44000
+    },
+    {
+      "epoch": 6.681681681681682,
+      "grad_norm": 26.814804077148438,
+      "learning_rate": 6.642642642642643e-06,
+      "loss": 0.2544,
+      "step": 44500
+    },
+    {
+      "epoch": 6.756756756756757,
+      "grad_norm": 3.904442548751831,
+      "learning_rate": 6.4924924924924924e-06,
+      "loss": 0.2322,
+      "step": 45000
+    },
+    {
+      "epoch": 6.831831831831832,
+      "grad_norm": 0.3679012358188629,
+      "learning_rate": 6.342642642642643e-06,
+      "loss": 0.2314,
+      "step": 45500
+    },
+    {
+      "epoch": 6.906906906906907,
+      "grad_norm": 0.13983987271785736,
+      "learning_rate": 6.192492492492494e-06,
+      "loss": 0.2559,
+      "step": 46000
+    },
+    {
+      "epoch": 6.981981981981982,
+      "grad_norm": 13.696492195129395,
+      "learning_rate": 6.042342342342343e-06,
+      "loss": 0.2535,
+      "step": 46500
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.91875,
+      "eval_f1": 0.9199267521225237,
+      "eval_loss": 0.2686730921268463,
+      "eval_precision": 0.9094799210006583,
+      "eval_recall": 0.9306163691478613,
+      "eval_runtime": 6.8805,
+      "eval_samples_per_second": 860.409,
+      "eval_steps_per_second": 107.551,
+      "step": 46620
+    },
+    {
+      "epoch": 7.057057057057057,
+      "grad_norm": 1.2342840433120728,
+      "learning_rate": 5.892192192192193e-06,
+      "loss": 0.2461,
+      "step": 47000
+    },
+    {
+      "epoch": 7.132132132132132,
+      "grad_norm": 0.11590774357318878,
+      "learning_rate": 5.742042042042042e-06,
+      "loss": 0.2341,
+      "step": 47500
+    },
+    {
+      "epoch": 7.207207207207207,
+      "grad_norm": 0.7022895216941833,
+      "learning_rate": 5.592192192192193e-06,
+      "loss": 0.2617,
+      "step": 48000
+    },
+    {
+      "epoch": 7.282282282282282,
+      "grad_norm": 0.6305549740791321,
+      "learning_rate": 5.442042042042043e-06,
+      "loss": 0.2379,
+      "step": 48500
+    },
+    {
+      "epoch": 7.357357357357357,
+      "grad_norm": 1.8172821998596191,
+      "learning_rate": 5.291891891891892e-06,
+      "loss": 0.2402,
+      "step": 49000
+    },
+    {
+      "epoch": 7.4324324324324325,
+      "grad_norm": 5.735633373260498,
+      "learning_rate": 5.1417417417417425e-06,
+      "loss": 0.2468,
+      "step": 49500
+    },
+    {
+      "epoch": 7.5075075075075075,
+      "grad_norm": 0.6238136291503906,
+      "learning_rate": 4.991891891891893e-06,
+      "loss": 0.2284,
+      "step": 50000
+    },
+    {
+      "epoch": 7.5825825825825826,
+      "grad_norm": 1.6428614854812622,
+      "learning_rate": 4.841741741741742e-06,
+      "loss": 0.2242,
+      "step": 50500
+    },
+    {
+      "epoch": 7.657657657657658,
+      "grad_norm": 3.6709420680999756,
+      "learning_rate": 4.691591591591592e-06,
+      "loss": 0.2313,
+      "step": 51000
+    },
+    {
+      "epoch": 7.732732732732733,
+      "grad_norm": 1.36298406124115,
+      "learning_rate": 4.541441441441442e-06,
+      "loss": 0.2372,
+      "step": 51500
+    },
+    {
+      "epoch": 7.807807807807808,
+      "grad_norm": 1.058044672012329,
+      "learning_rate": 4.391291291291292e-06,
+      "loss": 0.2267,
+      "step": 52000
+    },
+    {
+      "epoch": 7.882882882882883,
+      "grad_norm": 9.829523086547852,
+      "learning_rate": 4.241441441441442e-06,
+      "loss": 0.2695,
+      "step": 52500
+    },
+    {
+      "epoch": 7.957957957957958,
+      "grad_norm": 6.040225982666016,
+      "learning_rate": 4.091591591591592e-06,
+      "loss": 0.2359,
+      "step": 53000
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.9175675675675675,
+      "eval_f1": 0.9189099368560983,
+      "eval_loss": 0.2778928279876709,
+      "eval_precision": 0.9068547064611348,
+      "eval_recall": 0.9312899966318626,
+      "eval_runtime": 6.9239,
+      "eval_samples_per_second": 855.009,
+      "eval_steps_per_second": 106.876,
+      "step": 53280
+    },
+    {
+      "epoch": 8.033033033033034,
+      "grad_norm": 0.8012737035751343,
+      "learning_rate": 3.941441441441442e-06,
+      "loss": 0.2353,
+      "step": 53500
+    },
+    {
+      "epoch": 8.108108108108109,
+      "grad_norm": 33.150291442871094,
+      "learning_rate": 3.7912912912912915e-06,
+      "loss": 0.2411,
+      "step": 54000
+    },
+    {
+      "epoch": 8.183183183183184,
+      "grad_norm": 0.2979067265987396,
+      "learning_rate": 3.6411411411411413e-06,
+      "loss": 0.2546,
+      "step": 54500
+    },
+    {
+      "epoch": 8.258258258258259,
+      "grad_norm": 0.2562500536441803,
+      "learning_rate": 3.490990990990991e-06,
+      "loss": 0.2189,
+      "step": 55000
+    },
+    {
+      "epoch": 8.333333333333334,
+      "grad_norm": 0.5810413360595703,
+      "learning_rate": 3.340840840840841e-06,
+      "loss": 0.2389,
+      "step": 55500
+    },
+    {
+      "epoch": 8.408408408408409,
+      "grad_norm": 15.872923851013184,
+      "learning_rate": 3.190690690690691e-06,
+      "loss": 0.2512,
+      "step": 56000
+    },
+    {
+      "epoch": 8.483483483483484,
+      "grad_norm": 1.3113569021224976,
+      "learning_rate": 3.040540540540541e-06,
+      "loss": 0.2436,
+      "step": 56500
+    },
+    {
+      "epoch": 8.558558558558559,
+      "grad_norm": 13.303168296813965,
+      "learning_rate": 2.890690690690691e-06,
+      "loss": 0.2288,
+      "step": 57000
+    },
+    {
+      "epoch": 8.633633633633634,
+      "grad_norm": 22.601531982421875,
+      "learning_rate": 2.7405405405405404e-06,
+      "loss": 0.2466,
+      "step": 57500
+    },
+    {
+      "epoch": 8.70870870870871,
+      "grad_norm": 0.9369896054267883,
+      "learning_rate": 2.5903903903903906e-06,
+      "loss": 0.2633,
+      "step": 58000
+    },
+    {
+      "epoch": 8.783783783783784,
+      "grad_norm": 10.832354545593262,
+      "learning_rate": 2.4402402402402404e-06,
+      "loss": 0.2297,
+      "step": 58500
+    },
+    {
+      "epoch": 8.85885885885886,
+      "grad_norm": 7.654428958892822,
+      "learning_rate": 2.2903903903903905e-06,
+      "loss": 0.2336,
+      "step": 59000
+    },
+    {
+      "epoch": 8.933933933933934,
+      "grad_norm": 20.344663619995117,
+      "learning_rate": 2.1402402402402403e-06,
+      "loss": 0.2386,
+      "step": 59500
+    },
+    {
+      "epoch": 9.0,
+      "eval_accuracy": 0.9175675675675675,
+      "eval_f1": 0.9182579564489112,
+      "eval_loss": 0.2806470990180969,
+      "eval_precision": 0.9133622125958014,
+      "eval_recall": 0.9232064668238464,
+      "eval_runtime": 6.8833,
+      "eval_samples_per_second": 860.047,
+      "eval_steps_per_second": 107.506,
+      "step": 59940
+    },
+    {
+      "epoch": 9.00900900900901,
+      "grad_norm": 8.42081069946289,
+      "learning_rate": 1.99009009009009e-06,
+      "loss": 0.2485,
+      "step": 60000
+    },
+    {
+      "epoch": 9.084084084084084,
+      "grad_norm": 25.98297119140625,
+      "learning_rate": 1.83993993993994e-06,
+      "loss": 0.2313,
+      "step": 60500
+    },
+    {
+      "epoch": 9.15915915915916,
+      "grad_norm": 19.054916381835938,
+      "learning_rate": 1.6900900900900902e-06,
+      "loss": 0.2339,
+      "step": 61000
+    },
+    {
+      "epoch": 9.234234234234235,
+      "grad_norm": 17.01715850830078,
+      "learning_rate": 1.53993993993994e-06,
+      "loss": 0.2264,
+      "step": 61500
+    },
+    {
+      "epoch": 9.30930930930931,
+      "grad_norm": 0.13271598517894745,
+      "learning_rate": 1.38978978978979e-06,
+      "loss": 0.2224,
+      "step": 62000
+    },
+    {
+      "epoch": 9.384384384384385,
+      "grad_norm": 0.999711275100708,
+      "learning_rate": 1.2396396396396398e-06,
+      "loss": 0.2321,
+      "step": 62500
+    },
+    {
+      "epoch": 9.45945945945946,
+      "grad_norm": 13.851149559020996,
+      "learning_rate": 1.0897897897897899e-06,
+      "loss": 0.2227,
+      "step": 63000
+    },
+    {
+      "epoch": 9.534534534534535,
+      "grad_norm": 0.29212486743927,
+      "learning_rate": 9.396396396396397e-07,
+      "loss": 0.2307,
+      "step": 63500
+    },
+    {
+      "epoch": 9.60960960960961,
+      "grad_norm": 2.285646438598633,
+      "learning_rate": 7.894894894894896e-07,
+      "loss": 0.2517,
+      "step": 64000
+    },
+    {
+      "epoch": 9.684684684684685,
+      "grad_norm": 0.19670137763023376,
+      "learning_rate": 6.393393393393393e-07,
+      "loss": 0.2697,
+      "step": 64500
+    },
+    {
+      "epoch": 9.75975975975976,
+      "grad_norm": 19.214984893798828,
+      "learning_rate": 4.891891891891892e-07,
+      "loss": 0.2662,
+      "step": 65000
+    },
+    {
+      "epoch": 9.834834834834835,
+      "grad_norm": 0.19054338335990906,
+      "learning_rate": 3.393393393393394e-07,
+      "loss": 0.217,
+      "step": 65500
+    },
+    {
+      "epoch": 9.90990990990991,
+      "grad_norm": 23.02910804748535,
+      "learning_rate": 1.8918918918918921e-07,
+      "loss": 0.2481,
+      "step": 66000
+    },
+    {
+      "epoch": 9.984984984984985,
+      "grad_norm": 0.41014552116394043,
+      "learning_rate": 3.903903903903904e-08,
+      "loss": 0.2379,
+      "step": 66500
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.9177364864864865,
+      "eval_f1": 0.9184391224250544,
+      "eval_loss": 0.2841491401195526,
+      "eval_precision": 0.9133910726182545,
+      "eval_recall": 0.9235432805658471,
+      "eval_runtime": 6.9645,
+      "eval_samples_per_second": 850.023,
+      "eval_steps_per_second": 106.253,
+      "step": 66600
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 66600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.4922753261568e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

04-gene-sft/gpt2_lora_text_classification/checkpoint-66600/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e641b388947ac47c75e10e6525af9bbc8a067f3617ad53397af08cf2c7e505
+size 5304