Upload folder using huggingface_hub

04-gene-sft/.ipynb_checkpoints/1-finetue-intro-checkpoint.ipynb

@@ -31,6 +31,12 @@
     "\"yuanzhoulvpi/gpt2_chinese\", num_labels=2\n",
     ")\n",
     "\n",
+    "对应的训练数据一般是这样的：\n",
+    "\n",
+    "| seq                          | label |\n",
+    "|------------------------------|-------|\n",
+    "| 他家的奶茶超级好喝。。。      | 1     |\n",
+    "| 他家的奶茶超级难喝。。。      | 0     |\n",
     "\n",
     "\n",
     "2 如果是把分类问题，改成指令微调的模式，就是像\n",
@@ -174,7 +180,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "64312191-423f-4a18-aa0c-036374e93fb2",
    "metadata": {},
    "outputs": [],
@@ -192,10 +198,44 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "32c16282-f9f1-4545-b522-daf2b39b4ead",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2Model(\n",
+       "  (wte): Embedding(50257, 768)\n",
+       "  (wpe): Embedding(1024, 768)\n",
+       "  (drop): Dropout(p=0.1, inplace=False)\n",
+       "  (h): ModuleList(\n",
+       "    (0-11): 12 x GPT2Block(\n",
+       "      (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "      (attn): GPT2SdpaAttention(\n",
+       "        (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "        (c_proj): Conv1D(nf=768, nx=768)\n",
+       "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "      )\n",
+       "      (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "      (mlp): GPT2MLP(\n",
+       "        (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "        (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "        (act): NewGELUActivation()\n",
+       "        (dropout): Dropout(p=0.1, inplace=False)\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#原始模型\n",
     "from transformers import AutoModel\n",
@@ -205,10 +245,55 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "1149163f-4d89-472e-8d45-ebcbb5f9575e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "GPT2ForSequenceClassification(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (score): Linear(in_features=768, out_features=2, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#分类微调模型\n",
     "from transformers import AutoModelForSequenceClassification\n",
@@ -218,16 +303,61 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "id": "09735059-507c-48c4-893f-ca0da21ce5e8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#指令微调模型\n",
     "from transformers import AutoModelForCausalLM\n",
-    "sft_model = AutoModelForMaskedLM.from_pretrained(\"gpt2\")\n",
+    "sft_model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
     "sft_model"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1407cbe-4996-4898-a135-e26d28da2a2a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/2-gpt2-instruction-ft-checkpoint.ipynb

@@ -8,6 +8,123 @@
     "# 4.2 基于GPT2的指令微调"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "02cd6e13-bbfb-413a-8236-ff092456fd1c",
+   "metadata": {},
+   "source": [
+    "我还是用第二章中的分类的例子，使用指令微调的形式，来再次解决分类问题。\n",
+    "\n",
+    "使用 GPT-2 进行文本分类的两种方法：**使用 GPT-2 的分类头（Classification Header）** 和 **将分类任务转换为指令微调**，在思路、实现、优劣势和适用场景上存在明显差异。以下是详细对比：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 核心思路**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **基本概念**                | 在 GPT-2 顶部添加一个分类头（通常是一个线性层），直接预测分类标签。 | 将分类任务转化为自然语言指令，模型通过微调理解并完成指令形式的任务。 |\n",
+    "| **实现方式**                | 修改 GPT-2 模型，添加 `num_labels` 分类头并定义分类损失函数。     | 构建任务指令数据（Instruction + Input + Output），然后微调模型。 |\n",
+    "| **数据形式**                | 文本与其分类标签的直接映射。                                   | 文本通过指令转化为生成任务。例如：<br>`Input`: 文章内容<br>`Output`: 分类结果。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 数据格式**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **数据格式**                | - 输入：文本 <br>- 标签：离散类别标签（如 0, 1, 2）。             | - 指令：自然语言描述任务（如 \"请分类以下文本\"）。<br>- 输入：分类文本。<br>- 输出：分类结果（文本形式）。 |\n",
+    "| **示例**                   | 输入：`\"This is a happy day!\"`<br>标签：`1`（表示积极）         | `Instruction`: \"请对以下文本进行情感分类\"<br>`Input`: `\"This is a happy day!\"`<br>`Output`: `\"积极\"` |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 模型结构**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **模型结构**                | - GPT-2 + 分类头（线性层）。                                   | - GPT-2 原始结构，无需额外的分类头。                   |\n",
+    "| **损失函数**                | - 使用交叉熵损失（Cross Entropy Loss）。                       | - 使用自回归的语言建模损失（Language Modeling Loss）。  |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 训练过程**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **微调对象**                | 主要微调分类头部分的参数（可选择冻结 GPT-2 的主干部分）。         | 微调整个 GPT-2 模型（或使用参数高效微调如 LoRA）。      |\n",
+    "| **标签处理**                | 离散化标签（如 0, 1, 2）。                                    | 标签转化为自然语言（如“积极”、“中立”、“消极”）。      |\n",
+    "| **训练难度**                | - 简单，标准分类任务流程。<br>- 数据需求较小，适合小规模微调。     | - 复杂，需要构造高质量的指令数据集。<br>- 数据需求较大，适合多任务场景。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 优缺点分析**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **优点**                    | - 训练速度快，计算资源需求较低。<br>- 实现简单，适合单一任务。    | - 泛化能力��，支持多任务扩展。<br>- 与多任务微调和开放式生成兼容。 |\n",
+    "| **缺点**                    | - 只能处理分类任务，难以扩展为其他任务。<br>- 需要人工调整分类头和损失函数。 | - 数据构造复杂且对数据质量依赖较高。<br>- 训练资源需求较大，训练时间较长。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 适用场景**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **适用场景**                | - 单任务文本分类，如情感分析、垃圾邮件检测等。                 | - 多任务场景，支持分类、翻译、摘要等任务的统一处理。     |\n",
+    "| **数据规模**                | 适合小数据集，数千到数万条数据即可训练效果良好。                  | 适合大数据集，特别是多任务、多领域的数据集。             |\n",
+    "| **需求类型**                | 专注于提高单一任务的分类准确率。                                | 需要增强模型的多任务泛化能力，同时提升用户交互体验。     |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 综合对比总结**\n",
+    "\n",
+    "| **维度**                | **使用 GPT-2 分类头**                                           | **转换为指令微调**                                      |\n",
+    "|-------------------------|--------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **实现复杂度**          | 较低，直接添加分类头并使用标准分类流程即可完成。                    | 较高，需要构造高质量指令数据，并调整训练流程。            |\n",
+    "| **资源需求**            | 较低，仅需调整分类头部分，训练时间和显存消耗较少。                   | 较高，需要微调整个模型，且对数据和算力需求更大。          |\n",
+    "| **性能表现**            | 对单一分类任务效果较好，但泛化能力较弱。                           | 在多任务、多样化分类场景中表现更强，且可扩展为其他任务类型。 |\n",
+    "| **扩展性**              | 较差，仅适用于当前任务，难以迁移到其他任务。                        | 较强，可适应多任务指令和开放式生成场景。                 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **选择建议**\n",
+    "\n",
+    "1. **使用 GPT-2 分类头**：\n",
+    "   - 如果任务是单一分类问题（如情感分析、垃圾邮件检测），并且数据量有限，推荐使用分类头方法。\n",
+    "   - 适合快速实现和部署，无需复杂的预处理和指令数据集构建。\n",
+    "\n",
+    "2. **转换为指令微调**：\n",
+    "   - 如果任务需要多样化（分类+生成+翻译等），或需要对未见任务有更好的泛化能力，推荐使用指令微调。\n",
+    "   - 适合多任务、多场景部署，尤其是在 ChatGPT 风格的应用中更为适用。\n",
+    "\n",
+    "通过综合任务需求、数据规模和资源条件选择合适的方法，能够有效提升模型性能并实现更广泛的适用性。\n",
+    "\n",
+    "\n",
+    "原始的数据格式如下：\n",
+    "| sequence                                               | label | label_name     |\n",
+    "|--------------------------------------------------------|-------|----------------|\n",
+    "| TATATTTTCTCAGCTGAGTTAATTAGTTTCACTAGTTAACTGAGAATAAAAGAA | 1     | promoter       |\n",
+    "| TGGGGAGGGTCCGGTGTTAGTTAGATACATCCCCAGACCCACACCCCGGATAGA | 0     | Non-promoter   |\n",
+    "\n",
+    "转成指令的格式为：\n",
+    "```\n",
+    "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.', \n",
+    "'input': 'CATGCGGGTCG...', \n",
+    "'output': 'Non-promoter'}\n",
+    "```\n",
+    "\n",
+    "然后写成指令微调数据格式，当做一般的文本进行训练：\n",
+    "```\n",
+    "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+    "### Instruction:\n",
+    "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+    "### Input:\n",
+    "TCTTTCTCTTCTGTATCATTCTACTT...\n",
+    "### Response:\n",
+    "Non-promoter\n",
+    "```\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

04-gene-sft/.ipynb_checkpoints/3-llama-expand-dict-checkpoint.ipynb

@@ -114,10 +114,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "19a06b82-31b8-48cb-9c83-ec016da2da8a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['▁TCG', 'ACGGC', 'ACGCG', 'ACAGC', 'AGCG', 'AGCCCC', 'GCGC', 'ACCCG', 'AGCGCG', 'AKCG', 'FVGP', 'MV', 'HLKV', 'HLE', 'ADV', 'ASSC', 'RS', 'AVI', 'YL', 'TS', 'EEP', 'FEG', 'VLGL', 'RLKE', 'GI', 'AI', 'TGC', 'WPR', 'WP', 'DEM', 'DE', 'RS', 'AVW', 'RV', 'EPY', 'TR', 'HFG', 'RVL', 'YS', 'FGV']\n"
+     ]
+    }
+   ],
    "source": [
     "from sentencepiece import SentencePieceProcessor\n",
     "model_path = \"gene_bpe_seg.model\"\n",
@@ -147,7 +155,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "3bafcc33-2923-4026-bc39-c6ec716d2e3c",
    "metadata": {},
    "outputs": [],
@@ -161,10 +169,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "66cb86ed-3225-4bb0-8aca-6005bc918d03",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "32000 60000\n",
+      "['<s>', '</s>', '<unk>']\n",
+      "[1, 2, 0]\n",
+      "{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}\n"
+     ]
+    }
+   ],
    "source": [
     "llama_tokenizer_dir = \"llama-7b-hf\" \n",
     "dna_sp_model_file = \"gene_bpe_seg.model\"\n",
@@ -188,10 +214,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "7ba4240e-bc08-4be0-8ca3-c4e7a47fa055",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "32000\n",
+      "Before:32000\n",
+      "New model pieces: 91643\n"
+     ]
+    }
+   ],
    "source": [
     "## Add dna tokens to LLaMA tokenizer\n",
     "llama_spm_tokens_set=set(p.piece for p in llama_spm.pieces)\n",
@@ -210,10 +246,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "a240a7d8-c1a9-4473-a5c5-157a25f97c16",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gene-LLaMA tokenizer has been saved to merged_gene_eng_tokenizer_hf\n"
+     ]
+    }
+   ],
    "source": [
     "## Save\n",
     "output_sp_dir = 'merged_gene_eng_tokenizer_sp'\n",
@@ -229,10 +273,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "cbd1f648-f8a0-4f16-b516-2ce3e7c7cfee",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['<s>', '</s>', '<unk>']\n",
+      "[1, 2, 0]\n",
+      "{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}\n",
+      "Test text:\n",
+      " TCGACGGCACGCGACAGCAGCGAGCCCCGCGCACCCGAGCGCGAKCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+      "The primary use of LLaMA is research on large language models, including\n",
+      "Tokenized by LLaMA tokenizer:['▁T', 'CG', 'AC', 'G', 'GC', 'AC', 'GC', 'G', 'AC', 'AG', 'CA', 'GC', 'G', 'AG', 'CC', 'CC', 'GC', 'GC', 'AC', 'CC', 'GA', 'GC', 'GC', 'GA', 'K', 'CG', 'F', 'V', 'G', 'PM', 'V', 'HL', 'K', 'V', 'H', 'LE', 'AD', 'VA', 'SS', 'CR', 'S', 'AV', 'I', 'Y', 'LT', 'SEE', 'PF', 'EG', 'V', 'L', 'GL', 'RL', 'KE', 'G', 'IA', 'IT', 'GC', 'W', 'PR', 'WP', 'DE', 'MD', 'ERS', 'AV', 'WR', 'VE', 'PY', 'TR', 'H', 'F', 'GR', 'V', 'LY', 'SF', 'GV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n",
+      "Tokenized by GENE-LLaMA tokenizer:['▁TCG', 'ACGGC', 'ACGCG', 'ACAG', 'CA', 'GCG', 'AGCCCC', 'GCGC', 'ACCCG', 'AGCGCG', 'AKCG', 'FVGP', 'MVHL', 'KV', 'HLE', 'ADV', 'ASSC', 'RSAV', 'I', 'YL', 'TSEE', 'P', 'FEG', 'VLGL', 'RLK', 'EGI', 'AI', 'TGC', 'W', 'PRW', 'P', 'DEM', 'DER', 'SAV', 'W', 'RVE', 'PY', 'TRH', 'FG', 'RVLY', 'SFGV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n"
+     ]
+    }
+   ],
    "source": [
     "# Test\n",
     "llama_tokenizer = LlamaTokenizer.from_pretrained(llama_tokenizer_dir)\n",
@@ -246,6 +305,14 @@
     "print(f\"Tokenized by LLaMA tokenizer:{llama_tokenizer.tokenize(text)}\")\n",
     "print(f\"Tokenized by GENE-LLaMA tokenizer:{dna_llama_tokenizer.tokenize(text)}\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46ae7605-2ef8-4927-bff3-2c0325f8df0d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/4-deepspeed-intro-checkpoint.ipynb

@@ -56,6 +56,8 @@
     "\n",
     "每个阶段都进一步减少显存需求，Stage 3 可支持超大规模模型（如 GPT-3）。\n",
     "\n",
+    "<img src='img/deepspeed.png' width='600px' />\n",
+    "\n",
     "#### **（2）混合精度训练**\n",
     "通过 FP16 或 BF16（半精度浮点数）计算，显著减少显存占用并提升计算效率。\n",
     "\n",
@@ -567,6 +569,14 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce701aeb-c8c7-450a-bbf9-b793a19cd0c6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/.ipynb_checkpoints/5-peft-intro-checkpoint.ipynb

@@ -0,0 +1,870 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "963e9ae0-ac68-44be-8c7d-fb9842784362",
+   "metadata": {},
+   "source": [
+    "# 4.6 基于llama的基因大模型指令微调"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "182b82c4-d484-4c15-a600-03c3b51367ec",
+   "metadata": {},
+   "source": [
+    "**PEFT**（Parameter-Efficient Fine-Tuning，参数高效微调）是一种优化技术，旨在以最小的参数更新实现对大规模预训练模型（如 GPT、BERT 等）的微调。PEFT 技术通过减少微调所需的参数量，显著降低了存储和计算开销，同时保留模型的性能，特别适合资源受限的场景和领域特定任务的定制化。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 核心思想**\n",
+    "传统的微调方式需要更新整个预训练模型的所有参数，PEFT 技术通过只调整少量的参数（如特定层或额外添加的小型模块）实现微调目标，大幅减少了训练开销和存储需求。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 常见的 PEFT 方法**\n",
+    "\n",
+    "#### **（1）Adapter 模型**\n",
+    "- 在每一层 Transformer 的输出中插入小型适配器模块，仅训练适配器模块的参数。\n",
+    "- 原始模型参数保持冻结不变。\n",
+    "- 优点：适配器模块参数量小，能适应不同任务。\n",
+    "\n",
+    "示例方法：\n",
+    "- **AdapterFusion**\n",
+    "- **MAD-X**\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（2）Prefix Tuning**\n",
+    "- 在 Transformer 的输入前添加一组可学习的前缀向量，这些前缀与模型的注意力机制交互。\n",
+    "- 只调整前缀向量的参数，而不更新原始模型。\n",
+    "- 优点：对生成任务效果显著，参数量进一步减少。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（3）LoRA（Low-Rank Adaptation）**\n",
+    "- 将预训练模型中的部分权重分解为两个低秩矩阵，仅调整这些低秩矩阵的参数。\n",
+    "- 原始权重保持冻结状态。\n",
+    "- 优点：参数量极小，计算高效。\n",
+    "  \n",
+    "---\n",
+    "\n",
+    "#### **（4）Prompt Tuning**\n",
+    "- 在输入文本中添加可学习的提示（Prompt）。\n",
+    "- 适合 NLP 任务中的文本生成、分类等。\n",
+    "- 优点：实现简单，易于集成到现有框架。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. PEFT 的优势**\n",
+    "\n",
+    "1. **显著减少参数更新量**：\n",
+    "   - 微调传统的大模型（如 GPT-3）需要更新数百亿参数，而 PEFT 仅需更新百万级别甚至更少的参数。\n",
+    "\n",
+    "2. **高效存储**：\n",
+    "   - 每个任务的微调结果只需存储少量额外参数，而不是整个模型。\n",
+    "\n",
+    "3. **适用多任务**：\n",
+    "   - 同一预训练模型可以通过不同的 PEFT 模块适配多个任务，无需重新训练。\n",
+    "\n",
+    "4. **降低计算开销**：\n",
+    "   - 训练所需的内存和计算显著减少，适合资源有限的环境。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 应用场景**\n",
+    "\n",
+    "1. **领域特定任务**：\n",
+    "   - 医疗、法律、金融等领域微调预训练模型。\n",
+    "\n",
+    "2. **多任务学习**：\n",
+    "   - 适配多个任务，复用同一模型的预训练权重。\n",
+    "\n",
+    "3. **资源受限场景**：\n",
+    "   - 移动设备、边缘设备上的模型部署。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. Hugging Face PEFT 库**\n",
+    "\n",
+    "Hugging Face 提供了专门的 PEFT 库，支持多种参数高效微调技术：\n",
+    "- **安装**：\n",
+    "  ```bash\n",
+    "  pip install peft\n",
+    "  ```\n",
+    "- **使用 LoRA 微调示例**：\n",
+    "  ```python\n",
+    "  from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "  from peft import LoraConfig, get_peft_model, TaskType\n",
+    "\n",
+    "  # 加载模型和分词器\n",
+    "  model_name = \"gpt2\"\n",
+    "  model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "  tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "\n",
+    "  # 配置 LoRA\n",
+    "  lora_config = LoraConfig(\n",
+    "      task_type=TaskType.CAUSAL_LM,\n",
+    "      r=8,\n",
+    "      lora_alpha=32,\n",
+    "      target_modules=[\"q_proj\", \"v_proj\"],\n",
+    "      lora_dropout=0.1,\n",
+    "      bias=\"none\"\n",
+    "  )\n",
+    "\n",
+    "  # 使用 LoRA 微调模型\n",
+    "  model = get_peft_model(model, lora_config)\n",
+    "  model.print_trainable_parameters()\n",
+    "\n",
+    "  # 微调代码...\n",
+    "  ```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. PEFT 的局限性**\n",
+    "1. **特定任务限制**：\n",
+    "   - 在一些复杂任务中，PEFT 方法可能不如全量微调效果好。\n",
+    "\n",
+    "2. **需要设计合适的模块**：\n",
+    "   - 不同任务需要选择和设计合��的 PEFT 技术。\n",
+    "\n",
+    "3. **与模型架构相关**：\n",
+    "   - PEFT 技术可能需要对模型架构进行一定程度的修改。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 小结**\n",
+    "PEFT 是一个极具潜力的技术，特别适合在有限资源下对大模型进行微调。它在许多领域和任务中已显示出良好的效果，例如 LoRA 和 Adapter 模型已经成为高效微调的主流方法。\n",
+    "\n",
+    "如果您需要实现高效微调，可以结合 Hugging Face 的 PEFT 库快速上手。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5aa3d240-44e1-4811-8f61-d6ff2500a798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
+   "metadata": {},
+   "source": [
+    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "41a0c049-9134-4d89-aad0-1aa2241a9fca",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4becc479adbc472bb7672d49da16aafd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "transformer\n",
+      "transformer.wte\n",
+      "transformer.wpe\n",
+      "transformer.drop\n",
+      "transformer.h\n",
+      "transformer.h.0\n",
+      "transformer.h.0.ln_1\n",
+      "transformer.h.0.attn\n",
+      "transformer.h.0.attn.c_attn\n",
+      "transformer.h.0.attn.c_proj\n",
+      "transformer.h.0.attn.attn_dropout\n",
+      "transformer.h.0.attn.resid_dropout\n",
+      "transformer.h.0.ln_2\n",
+      "transformer.h.0.mlp\n",
+      "transformer.h.0.mlp.c_fc\n",
+      "transformer.h.0.mlp.c_proj\n",
+      "transformer.h.0.mlp.act\n",
+      "transformer.h.0.mlp.dropout\n",
+      "transformer.h.1\n",
+      "transformer.h.1.ln_1\n",
+      "transformer.h.1.attn\n",
+      "transformer.h.1.attn.c_attn\n",
+      "transformer.h.1.attn.c_proj\n",
+      "transformer.h.1.attn.attn_dropout\n",
+      "transformer.h.1.attn.resid_dropout\n",
+      "transformer.h.1.ln_2\n",
+      "transformer.h.1.mlp\n",
+      "transformer.h.1.mlp.c_fc\n",
+      "transformer.h.1.mlp.c_proj\n",
+      "transformer.h.1.mlp.act\n",
+      "transformer.h.1.mlp.dropout\n",
+      "transformer.h.2\n",
+      "transformer.h.2.ln_1\n",
+      "transformer.h.2.attn\n",
+      "transformer.h.2.attn.c_attn\n",
+      "transformer.h.2.attn.c_proj\n",
+      "transformer.h.2.attn.attn_dropout\n",
+      "transformer.h.2.attn.resid_dropout\n",
+      "transformer.h.2.ln_2\n",
+      "transformer.h.2.mlp\n",
+      "transformer.h.2.mlp.c_fc\n",
+      "transformer.h.2.mlp.c_proj\n",
+      "transformer.h.2.mlp.act\n",
+      "transformer.h.2.mlp.dropout\n",
+      "transformer.h.3\n",
+      "transformer.h.3.ln_1\n",
+      "transformer.h.3.attn\n",
+      "transformer.h.3.attn.c_attn\n",
+      "transformer.h.3.attn.c_proj\n",
+      "transformer.h.3.attn.attn_dropout\n",
+      "transformer.h.3.attn.resid_dropout\n",
+      "transformer.h.3.ln_2\n",
+      "transformer.h.3.mlp\n",
+      "transformer.h.3.mlp.c_fc\n",
+      "transformer.h.3.mlp.c_proj\n",
+      "transformer.h.3.mlp.act\n",
+      "transformer.h.3.mlp.dropout\n",
+      "transformer.h.4\n",
+      "transformer.h.4.ln_1\n",
+      "transformer.h.4.attn\n",
+      "transformer.h.4.attn.c_attn\n",
+      "transformer.h.4.attn.c_proj\n",
+      "transformer.h.4.attn.attn_dropout\n",
+      "transformer.h.4.attn.resid_dropout\n",
+      "transformer.h.4.ln_2\n",
+      "transformer.h.4.mlp\n",
+      "transformer.h.4.mlp.c_fc\n",
+      "transformer.h.4.mlp.c_proj\n",
+      "transformer.h.4.mlp.act\n",
+      "transformer.h.4.mlp.dropout\n",
+      "transformer.h.5\n",
+      "transformer.h.5.ln_1\n",
+      "transformer.h.5.attn\n",
+      "transformer.h.5.attn.c_attn\n",
+      "transformer.h.5.attn.c_proj\n",
+      "transformer.h.5.attn.attn_dropout\n",
+      "transformer.h.5.attn.resid_dropout\n",
+      "transformer.h.5.ln_2\n",
+      "transformer.h.5.mlp\n",
+      "transformer.h.5.mlp.c_fc\n",
+      "transformer.h.5.mlp.c_proj\n",
+      "transformer.h.5.mlp.act\n",
+      "transformer.h.5.mlp.dropout\n",
+      "transformer.h.6\n",
+      "transformer.h.6.ln_1\n",
+      "transformer.h.6.attn\n",
+      "transformer.h.6.attn.c_attn\n",
+      "transformer.h.6.attn.c_proj\n",
+      "transformer.h.6.attn.attn_dropout\n",
+      "transformer.h.6.attn.resid_dropout\n",
+      "transformer.h.6.ln_2\n",
+      "transformer.h.6.mlp\n",
+      "transformer.h.6.mlp.c_fc\n",
+      "transformer.h.6.mlp.c_proj\n",
+      "transformer.h.6.mlp.act\n",
+      "transformer.h.6.mlp.dropout\n",
+      "transformer.h.7\n",
+      "transformer.h.7.ln_1\n",
+      "transformer.h.7.attn\n",
+      "transformer.h.7.attn.c_attn\n",
+      "transformer.h.7.attn.c_proj\n",
+      "transformer.h.7.attn.attn_dropout\n",
+      "transformer.h.7.attn.resid_dropout\n",
+      "transformer.h.7.ln_2\n",
+      "transformer.h.7.mlp\n",
+      "transformer.h.7.mlp.c_fc\n",
+      "transformer.h.7.mlp.c_proj\n",
+      "transformer.h.7.mlp.act\n",
+      "transformer.h.7.mlp.dropout\n",
+      "transformer.h.8\n",
+      "transformer.h.8.ln_1\n",
+      "transformer.h.8.attn\n",
+      "transformer.h.8.attn.c_attn\n",
+      "transformer.h.8.attn.c_proj\n",
+      "transformer.h.8.attn.attn_dropout\n",
+      "transformer.h.8.attn.resid_dropout\n",
+      "transformer.h.8.ln_2\n",
+      "transformer.h.8.mlp\n",
+      "transformer.h.8.mlp.c_fc\n",
+      "transformer.h.8.mlp.c_proj\n",
+      "transformer.h.8.mlp.act\n",
+      "transformer.h.8.mlp.dropout\n",
+      "transformer.h.9\n",
+      "transformer.h.9.ln_1\n",
+      "transformer.h.9.attn\n",
+      "transformer.h.9.attn.c_attn\n",
+      "transformer.h.9.attn.c_proj\n",
+      "transformer.h.9.attn.attn_dropout\n",
+      "transformer.h.9.attn.resid_dropout\n",
+      "transformer.h.9.ln_2\n",
+      "transformer.h.9.mlp\n",
+      "transformer.h.9.mlp.c_fc\n",
+      "transformer.h.9.mlp.c_proj\n",
+      "transformer.h.9.mlp.act\n",
+      "transformer.h.9.mlp.dropout\n",
+      "transformer.h.10\n",
+      "transformer.h.10.ln_1\n",
+      "transformer.h.10.attn\n",
+      "transformer.h.10.attn.c_attn\n",
+      "transformer.h.10.attn.c_proj\n",
+      "transformer.h.10.attn.attn_dropout\n",
+      "transformer.h.10.attn.resid_dropout\n",
+      "transformer.h.10.ln_2\n",
+      "transformer.h.10.mlp\n",
+      "transformer.h.10.mlp.c_fc\n",
+      "transformer.h.10.mlp.c_proj\n",
+      "transformer.h.10.mlp.act\n",
+      "transformer.h.10.mlp.dropout\n",
+      "transformer.h.11\n",
+      "transformer.h.11.ln_1\n",
+      "transformer.h.11.attn\n",
+      "transformer.h.11.attn.c_attn\n",
+      "transformer.h.11.attn.c_proj\n",
+      "transformer.h.11.attn.attn_dropout\n",
+      "transformer.h.11.attn.resid_dropout\n",
+      "transformer.h.11.ln_2\n",
+      "transformer.h.11.mlp\n",
+      "transformer.h.11.mlp.c_fc\n",
+      "transformer.h.11.mlp.c_proj\n",
+      "transformer.h.11.mlp.act\n",
+      "transformer.h.11.mlp.dropout\n",
+      "transformer.ln_f\n",
+      "lm_head\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForCausalLM\n",
+    "\n",
+    "model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
+    "\n",
+    "# 打印所有模块名称\n",
+    "for name, module in model.named_modules():\n",
+    "    print(name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37aa6abb-ab1c-4e9c-b968-579dd74044db",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0add2f79-f35c-4638-80bb-0d8a87a9b6a7",
+   "metadata": {},
+   "source": [
+    "在选择 `target_modules` 时，通常会根据模块的名称选择模型的特定部分，通常使用列表中最后一个点 `.` 后的字段名或整个路径名（如果需要更精确）。以下是对这些模块的详细分析和选择建议：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 分析模块结构**\n",
+    "\n",
+    "从列表中可以看出，GPT-2 的模块层次分为以下几类：\n",
+    "\n",
+    "1. **Embedding 层**：\n",
+    "   - `transformer.wte`：词嵌入层（Word Token Embeddings）。\n",
+    "   - `transformer.wpe`：位置嵌入层（Position Embeddings）。\n",
+    "\n",
+    "2. **Transformer 编码器层**：\n",
+    "   - 每层编号为 `transformer.h.<层号>`（共 12 层）。\n",
+    "   - 每层中包含：\n",
+    "     - **层归一化**：\n",
+    "       - `transformer.h.<层号>.ln_1`：第一层归一化。\n",
+    "       - `transformer.h.<层号>.ln_2`：第二层归一化。\n",
+    "     - **自注意力模块**：\n",
+    "       - `transformer.h.<层号>.attn.c_attn`：注意力模块的 Query、Key 和 Value 投影。\n",
+    "       - `transformer.h.<层号>.attn.c_proj`：注意力的输出投影。\n",
+    "       - `transformer.h.<层号>.attn.attn_dropout`：注意力的 Dropout。\n",
+    "       - `transformer.h.<层号>.attn.resid_dropout`：残差的 Dropout。\n",
+    "     - **前馈网络模块（MLP）**：\n",
+    "       - `transformer.h.<层号>.mlp.c_fc`：MLP 的第一层全连接。\n",
+    "       - `transformer.h.<层号>.mlp.c_proj`：MLP 的第二层全连接（输出投影）。\n",
+    "       - `transformer.h.<层号>.mlp.act`：激活函数（如 GELU）。\n",
+    "       - `transformer.h.<层号>.mlp.dropout`：MLP 的 Dropout。\n",
+    "\n",
+    "3. **最终层**：\n",
+    "   - `transformer.ln_f`：最终层归一化（LayerNorm）。\n",
+    "   - `lm_head`：语言建模头，用于生成预测的 token 分布。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 如何选择 `target_modules`**\n",
+    "\n",
+    "#### **（1）常见目标模块**\n",
+    "- `transformer.h.<层号>.attn.c_attn`：对自注意力模块的 Query、Key 和 Value 投影层微调。\n",
+    "- `transformer.h.<层号>.attn.c_proj`：对注意力输出的投影层微调。\n",
+    "- `transformer.h.<层号>.mlp.c_fc`：对前馈网络的输入全连接层微调。\n",
+    "- `transformer.h.<层号>.mlp.c_proj`：对前馈网络的输出投影层微调。\n",
+    "\n",
+    "#### **（2）推荐设置**\n",
+    "- **文本生成任务**：\n",
+    "  ```python\n",
+    "  target_modules = [\"transformer.h.*.attn.c_attn\", \"transformer.h.*.attn.c_proj\"]\n",
+    "  ```\n",
+    "  解释：\n",
+    "  - `*.attn.c_attn`：调整 Query、Key、Value 的生成。\n",
+    "  - `*.attn.c_proj`：调整注意力输出。\n",
+    "\n",
+    "- **文本分类任务**：\n",
+    "  ```python\n",
+    "  target_modules = [\"transformer.h.*.attn.c_attn\"]\n",
+    "  ```\n",
+    "  解释：\n",
+    "  - 微调自注意力模块最重要的部分即可。\n",
+    "\n",
+    "- **特定任务需要更细粒度控制**：\n",
+    "  - 仅微调某几层：\n",
+    "    ```python\n",
+    "    target_modules = [\"transformer.h.0.attn.c_attn\", \"transformer.h.0.mlp.c_fc\"]\n",
+    "    ```\n",
+    "\n",
+    "#### **（3）通配符选择**\n",
+    "使用 `*` 通配符可以指定所有层的某些模块：\n",
+    "- `transformer.h.*.attn.c_attn`：所有层的 Query、Key 和 Value 投影。\n",
+    "- `transformer.h.*.mlp.*`：所有层的 MLP 模块。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 示例：指定多个模块**\n",
+    "\n",
+    "```python\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.CAUSAL_LM,\n",
+    "    r=8,\n",
+    "    lora_alpha=32,\n",
+    "    target_modules=[\n",
+    "        \"transformer.h.*.attn.c_attn\",\n",
+    "        \"transformer.h.*.mlp.c_fc\"\n",
+    "    ],\n",
+    "    lora_dropout=0.1,\n",
+    "    bias=\"none\"\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "- 这表示对所有层的 `attn.c_attn` 和 `mlp.c_fc` 模块进行 LoRA 微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 小提示：如何确定适合的模块**\n",
+    "\n",
+    "1. **任务相关性**：\n",
+    "   - 文本生成：优先选择自注意力模块（如 `c_attn`）。\n",
+    "   - 文本分类：通常需要全局语义表示，选择 `attn.c_attn` 或 `mlp.c_fc`。\n",
+    "\n",
+    "2. **性能与资源平衡**：\n",
+    "   - 如果显存有限，可以只微调部分层。例如，仅选择浅层和深层的模块：\n",
+    "     ```python\n",
+    "     target_modules = [\"transformer.h.0.attn.c_attn\", \"transformer.h.11.attn.c_attn\"]\n",
+    "     ```\n",
+    "\n",
+    "3. **打印模块名称以调试**：\n",
+    "   - 确保选择的 `target_modules` 在模型中实际存在：\n",
+    "     ```python\n",
+    "     for name, _ in model.named_modules():\n",
+    "         if \"c_attn\" in name:\n",
+    "             print(name)\n",
+    "     ```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **建议**\n",
+    "- 一般情况下，`c_attn` 和 `c_proj` 是首选模块。\n",
+    "- 使用 `transformer.h.*` 通配符可以轻松指定多层。\n",
+    "- 根据任务需求和资源限制灵活调整目标模块，以实现最佳性能和效率。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b4a41750-420f-49c4-845d-69db394794f9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10c99eb9-8007-4297-972e-7be71768c9c3",
+   "metadata": {},
+   "source": [
+    "以下是对 `LoraConfig` 配置的更详细解释，特别是如何设置微调哪些参数、冻结哪些参数，以及一般如何选择这些设置：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. `LoraConfig` 参数解析**\n",
+    "\n",
+    "```python\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.SEQ_CLS,  # 序列分类任务\n",
+    "    r=8,                         # 降低矩阵秩\n",
+    "    lora_alpha=32,               # LoRA 的 alpha 超参数\n",
+    "    target_modules=[\"c_attn\"],   # GPT-2 中的自注意力模块\n",
+    "    lora_dropout=0.1,            # dropout 概率\n",
+    "    bias=\"none\",                 # 是否微调偏置参数\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "#### **（1）`task_type`**\n",
+    "- 定义任务类型，用于指导 PEFT 的具体行为。\n",
+    "- **常见选项**：\n",
+    "  - `TaskType.CAUSAL_LM`：自回归语言建模（如 GPT 系列模型）。\n",
+    "  - `TaskType.SEQ_CLS`：序列分类（如情感分析）。\n",
+    "  - `TaskType.TOKEN_CLS`：标注任务（如命名实体识别）。\n",
+    "  - `TaskType.SEQ_2_SEQ_LM`：序列到序列任务（如翻译、摘要）。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `TaskType.SEQ_CLS` 表示目标是文本分类任务。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（2）`r`**\n",
+    "- 表示 LoRA 的 **秩**（rank），是降低矩阵秩的核心参数。\n",
+    "- LoRA 通过将模型的权重分解为两个低秩矩阵（`A` 和 `B`），只更新这两个矩阵。\n",
+    "- `r` 的值越大，微调能力越强，但需要的额外参数也越多。\n",
+    "- **典型范围**：`4` 至 `64`，大多数任务中 `8` 或 `16` 是常用值。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `r=8` 表示使用低秩分解，并微调 8 维的参数矩阵。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（3）`lora_alpha`**\n",
+    "- 是 LoRA 的一个缩放因子，用于调节两个低秩矩阵的更新速率。\n",
+    "- **公式**：实际更新 = LoRA 输出 × `lora_alpha / r`\n",
+    "- **典型范围**：`16` 至 `128`，较大任务中可以选择更高的值。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `lora_alpha=32`，表示适中幅度的更新速率。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（4）`target_modules`**\n",
+    "- 指定要应用 LoRA 微调的模块。\n",
+    "- **常见选择**：\n",
+    "  - 对 Transformer 模型中的 **注意力模块**（如 `query`、`key`、`value`）进行微调，因为这些模块对任务性能影响较大。\n",
+    "  - 对 GPT-2，通常选择 `c_attn`（GPT-2 中负责自注意力机制的组合模块）。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `target_modules=[\"c_attn\"]` 表示只对 GPT-2 的自注意力模块 `c_attn` 应用 LoRA。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（5）`lora_dropout`**\n",
+    "- 表示 LoRA 层的 dropout 概率，用于防止过拟合。\n",
+    "- **典型范围**：`0.0` 至 `0.1`，视任务复杂性而定。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `lora_dropout=0.1`，表示有 10% 的概率随机丢弃 LoRA 层的输出。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（6）`bias`**\n",
+    "- 决定是否微调偏置参数。\n",
+    "- **选项**：\n",
+    "  - `\"none\"`：不微调任何偏置。\n",
+    "  - `\"all\"`：微调所有偏置。\n",
+    "  - `\"lora_only\"`：只微调 LoRA 层的偏置。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `bias=\"none\"`，表示所有偏置参数保持冻结。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 微调哪些参数，冻结哪些参数**\n",
+    "\n",
+    "LoRA 的核心思想是通过 **分解矩阵**，只更新少量参数，而冻结模型的大部分参数。以下是常见设置的说明：\n",
+    "\n",
+    "#### **微调的参数**\n",
+    "- LoRA 通过 `target_modules` 指定的模块，例如：\n",
+    "  - GPT-2 的 `c_attn`（自注意力模块）。\n",
+    "  - BERT 的 `query` 和 `key`。\n",
+    "- 这些模块是模型中对性能贡献最大的部分，通过微调这些模块，任务性能可以显著提升。\n",
+    "\n",
+    "#### **冻结的参数**\n",
+    "- 除了 `target_modules` 中指定的参数外，所有其他模型参数默认冻结，包括：\n",
+    "  - 预训练权重的绝大部分。\n",
+    "  - 偏置参数（如果 `bias=\"none\"`）。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 一般如何设置**\n",
+    "\n",
+    "#### **（1）针对不同任务调整**\n",
+    "- **文本分类任务**：\n",
+    "  - 优先选择自注意力模块（如 `c_attn`）作为 `target_modules`。\n",
+    "  - `r=8` 或 `r=16` 是常见选择，适中计算开销。\n",
+    "  - 设置适当的 dropout（如 `lora_dropout=0.1`）以防止过拟合。\n",
+    "  \n",
+    "- **语言生成任务**：\n",
+    "  - 对 GPT-2 或 GPT-3，选择 `q_proj` 和 `v_proj`（query 和 value 投影模块）。\n",
+    "  - `r=16` 或更高，适应生成任务的高复杂性。\n",
+    "\n",
+    "- **命名实体识别任务**：\n",
+    "  - 优先选择 `q_proj` 和 `k_proj`（query 和 key 模块）。\n",
+    "\n",
+    "#### **（2）参数量与显存的权衡**\n",
+    "- 如果显存有限，减少 `r` 的值。\n",
+    "- 对小型任务，`r=4` 或 `r=8` 通常已经足够。\n",
+    "\n",
+    "#### **（3）偏置设置**\n",
+    "- 偏置参数的影响较小，在大多数情况下，可以选择 `bias=\"none\"` 保持冻结。\n",
+    "- 对非常依赖偏置的任务（如生成风格微调），可以尝试 `bias=\"lora_only\"`。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 示例：如何选择目标模块**\n",
+    "\n",
+    "#### **GPT-2**\n",
+    "对 GPT-2 来说，以下模块通常是微调的目标：\n",
+    "- **`c_attn`**：注意力模块的组��层。\n",
+    "- **`q_proj` 和 `v_proj`**：Query 和 Value 的线性投影。\n",
+    "\n",
+    "#### **BERT**\n",
+    "对 BERT 来说，以下模块通常是微调的目标：\n",
+    "- **`query`**：Attention 的 Query 模块。\n",
+    "- **`key`**：Attention 的 Key 模块。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 总结建议**\n",
+    "- **微调的参数**：优先选择模型中注意力相关模块。\n",
+    "- **冻结的参数**：大部分参数默认冻结以节省显存。\n",
+    "- **配置选择**：根据任务复杂性调整 `r` 和 `target_modules`。\n",
+    "- **推荐起点**：\n",
+    "  - 文本分类：`target_modules=[\"c_attn\"]`, `r=8`, `lora_dropout=0.1`。\n",
+    "  - 文本生成：`target_modules=[\"q_proj\", \"v_proj\"]`, `r=16`, `lora_dropout=0.1`。\n",
+    "\n",
+    "通过这些设置，LoRA 可以在参数量极小的情况下实现高效微调，适合各种任务场景。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer\n",
+    "from peft import LoraConfig, get_peft_model, TaskType\n",
+    "from datasets import load_dataset\n",
+    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
+    "\n",
+    "# **1. 加载模型和分词器**\n",
+    "model_name = \"gpt2\"  # 基础模型\n",
+    "num_labels = 2       # 二分类任务\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "tokenizer.pad_token = tokenizer.eos_token  # 设置 pad_token 为 eos_token\n",
+    "\n",
+    "# **2. 定义数据集**\n",
+    "# 示例数据集：dna_promoter_300\n",
+    "dataset = load_dataset(\"dnagpt/dna_promoter_300\")['train'].train_test_split(test_size=0.1)\n",
+    "\n",
+    "# **3. 数据预处理**\n",
+    "def preprocess_function(examples):\n",
+    "    examples['label'] = [int(item) for item in examples['label']]\n",
+    "    return tokenizer(\n",
+    "        examples[\"sequence\"], truncation=True, padding=\"max_length\", max_length=128\n",
+    "    )\n",
+    "\n",
+    "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
+    "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "\n",
+    "# **4. 划分数据集**\n",
+    "train_dataset = tokenized_datasets[\"train\"]\n",
+    "test_dataset = tokenized_datasets[\"test\"]\n",
+    "\n",
+    "# **5. 配置 LoRA**\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.SEQ_CLS,  # 序列分类任务\n",
+    "    r=8,                         # 降低矩阵秩\n",
+    "    lora_alpha=32,               # LoRA 的 alpha 超参数\n",
+    "    target_modules=[\"c_attn\"],   # GPT-2 中的自注意力模块\n",
+    "    lora_dropout=0.1,            # dropout 概率\n",
+    "    bias=\"none\",                 # 是否微调偏置参数\n",
+    ")\n",
+    "\n",
+    "# 使用 LoRA 包装模型\n",
+    "model = get_peft_model(model, lora_config)\n",
+    "model.print_trainable_parameters()  # 打印可训练的参数信息\n",
+    "\n",
+    "# **6. 计算指标**\n",
+    "def compute_metrics(eval_pred):\n",
+    "    predictions, labels = eval_pred\n",
+    "    preds = predictions.argmax(axis=-1)\n",
+    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=\"binary\")\n",
+    "    acc = accuracy_score(labels, preds)\n",
+    "    return {\"accuracy\": acc, \"precision\": precision, \"recall\": recall, \"f1\": f1}\n",
+    "\n",
+    "# **7. 定义训练参数**\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2_lora_text_classification\",  # 模型保存路径\n",
+    "    evaluation_strategy=\"epoch\",                 # 每个 epoch 评估一次\n",
+    "    save_strategy=\"epoch\",                       # 每个 epoch 保存一次\n",
+    "    learning_rate=2e-5,                          # 学习率\n",
+    "    per_device_train_batch_size=8,               # 每设备的批量大小\n",
+    "    per_device_eval_batch_size=8,                # 每设备评估的批量大小\n",
+    "    num_train_epochs=3,                          # 训练轮数\n",
+    "    weight_decay=0.01,                           # 权重衰减\n",
+    "    logging_dir=\"./logs\",                        # 日志路径\n",
+    "    fp16=True,                                   # 启用混合精度训练\n",
+    "    save_total_limit=2,                          # 保留最多两个检查点\n",
+    "    load_best_model_at_end=True,                 # 加载最佳模型\n",
+    "    metric_for_best_model=\"accuracy\",            # 根据准确率选择最佳模型\n",
+    "    greater_is_better=True,\n",
+    ")\n",
+    "\n",
+    "# **8. 定义 Trainer**\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=train_dataset,\n",
+    "    eval_dataset=test_dataset,\n",
+    "    tokenizer=tokenizer,\n",
+    "    compute_metrics=compute_metrics,\n",
+    ")\n",
+    "\n",
+    "# **9. 开始训练**\n",
+    "trainer.train()\n",
+    "\n",
+    "# **10. 保存模型**\n",
+    "model.save_pretrained(\"./gpt2_lora_text_classification\")\n",
+    "tokenizer.save_pretrained(\"./gpt2_lora_text_classification\")\n",
+    "\n",
+    "print(\"训练完成，模型已保存至 ./gpt2_lora_text_classification\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49a60fed-3a7d-4608-98b1-b4e313b94dbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+    "from peft import PeftModel\n",
+    "\n",
+    "# 加载分词器\n",
+    "model_path = \"./gpt2_lora_text_classification\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
+    "\n",
+    "# 加载微调后的 PEFT 模型\n",
+    "base_model = AutoModelForSequenceClassification.from_pretrained(\"gpt2\", num_labels=2)\n",
+    "model = PeftModel.from_pretrained(base_model, model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c0d8f02-c3dc-4961-8b3a-50eefc5f9448",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "\n",
+    "def predict(texts, model, tokenizer):\n",
+    "    \"\"\"\n",
+    "    使用微调后的 PEFT 模型进行推理。\n",
+    "    \n",
+    "    Args:\n",
+    "        texts (list of str): 待分类的文本列表。\n",
+    "        model (PeftModel): 微调后的模型。\n",
+    "        tokenizer (AutoTokenizer): 分词器。\n",
+    "    \n",
+    "    Returns:\n",
+    "        list of dict: 每个文本的预测结果，包括 logits 和预测的类别标签。\n",
+    "    \"\"\"\n",
+    "    # 对输入文本进行分词和编码\n",
+    "    inputs = tokenizer(\n",
+    "        texts,\n",
+    "        padding=True,\n",
+    "        truncation=True,\n",
+    "        max_length=512,\n",
+    "        return_tensors=\"pt\"\n",
+    "    )\n",
+    "    \n",
+    "    # 将输入数据移动到模型的设备上（CPU/GPU）\n",
+    "    inputs = {key: value.to(model.device) for key, value in inputs.items()}\n",
+    "    \n",
+    "    # 模型推理\n",
+    "    model.eval()\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "    \n",
+    "    # 获取 logits 并计算预测类别\n",
+    "    logits = outputs.logits\n",
+    "    probs = torch.nn.functional.softmax(logits, dim=-1)\n",
+    "    predictions = torch.argmax(probs, dim=-1)\n",
+    "    \n",
+    "    # 返回每个文本的预测结果\n",
+    "    results = [\n",
+    "        {\"text\": text, \"logits\": logit.tolist(), \"predicted_class\": int(pred)}\n",
+    "        for text, logit, pred in zip(texts, logits, predictions)\n",
+    "    ]\n",
+    "    return results\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9c0cfe65-f4f3-4274-a4f4-1ac13725b15a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Text: This movie was fantastic! I loved every part of it.\n",
+    "Predicted Class: 1\n",
+    "Logits: [-2.345, 3.567]\n",
+    "\n",
+    "Text: The plot was terrible and the acting was worse.\n",
+    "Predicted Class: 0\n",
+    "Logits: [4.123, -1.234]\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/.ipynb_checkpoints/6-llama-continue-train-checkpoint.ipynb

@@ -0,0 +1,491 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1e6d4978-4f0f-4268-aa23-d864857bd6c8",
+   "metadata": {},
+   "source": [
+    "# 4.6 基于llama的基因大模型持续预训练"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c201732-e736-463c-8446-637bf517479f",
+   "metadata": {},
+   "source": [
+    "LLaMA（**Large Language Model Meta AI**）是由 Meta（Facebook）开发的一系列大型语言模型，专注于提供高性能和高效的大语言模型，面向学术研究和开发社区。LLaMA 系列主要强调训练效率、模型性能和对计算资源的高效利用，是 GPT 系列模型的有力竞争者之一。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. LLaMA 模型概述**\n",
+    "\n",
+    "#### **1.1 LLaMA 1**\n",
+    "- **发布**：2023 年 2 月。\n",
+    "- **模型参数规模**：\n",
+    "  - 7B（70 亿）\n",
+    "  - 13B（130 亿）\n",
+    "  - 33B（330 亿）\n",
+    "  - 65B（650 亿）\n",
+    "- **特点**：\n",
+    "  - 专注于效率：与 GPT-3 等模型相比，LLaMA 在相同的训练成本下实现了更高的性能。\n",
+    "  - 针对研究开放：提供预训练模型权重供研究使用。\n",
+    "  - 使用高质量的数据：模型训练使用大量从网络中筛选的高质量文本数据，包括维基百科、书籍和其他高质量来源。\n",
+    "- **性能**：\n",
+    "  - 在许多 NLP 任务中，LLaMA 的性能超过 GPT-3 和其他同类模型。\n",
+    "  - 参数规模较小的版本（如 LLaMA-13B）性能可与 GPT-3（175B 参数）媲美。\n",
+    "\n",
+    "#### **1.2 LLaMA 2**\n",
+    "- **发布**：2023 年 7 月。\n",
+    "- **改进**：\n",
+    "  - 增强的训练数据：相比 LLaMA 1，使用了更多的高质量数据。\n",
+    "  - 引入微调版本：发布了开箱即用的对话模型（LLaMA 2-Chat）。\n",
+    "  - 更好的开源支持：LLaMA 2 在商业用途上比 LLaMA 1 更加开放。\n",
+    "- **模型参数规模**：\n",
+    "  - 7B（70 亿）\n",
+    "  - 13B（130 亿）\n",
+    "  - 70B（700 亿）\n",
+    "- **性能**：\n",
+    "  - LLaMA 2 的性能相比 LLaMA 1 有显著提升。\n",
+    "  - LLaMA 2-Chat 在对话任务中的表现优于许多现有开源模型。\n",
+    "  - 在多个标准基准（如 MMLU）上超过 GPT-4 和 Claude 的开源实现。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. LLaMA 的关键技术特点**\n",
+    "\n",
+    "#### **2.1 高效的架构设计**\n",
+    "- 基于 Transformer 架构。\n",
+    "- 针对训练效率和推理速度进行了优化，适合研究和开发。\n",
+    "\n",
+    "#### **2.2 模型压缩**\n",
+    "- 提供更小的参数规模（如 7B 和 13B），以便在更低的计算资源上运行。\n",
+    "- 在性能与参数量之间实现了很好的平衡。\n",
+    "\n",
+    "#### **2.3 训练数据**\n",
+    "- 使用从互联网中提取的高质量数据，注重数据清洗和筛选，避免低质量文本对模型的负面影响。\n",
+    "\n",
+    "#### **2.4 微调能力**\n",
+    "- 支持指令微调（Instruction Tuning）和 RLHF（基于人类反馈的强化学习），特别是在 LLaMA 2-Chat 模型中表现优异。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. LLaMA 的性能对比**\n",
+    "\n",
+    "#### **与 GPT-3 比较**\n",
+    "- LLaMA 1-13B 参数模型在许多任务上的性能接近 GPT-3-175B。\n",
+    "- LLaMA 2-70B 在多个任务上超过 GPT-3。\n",
+    "\n",
+    "#### **与其他开源模型比较**\n",
+    "- LLaMA 2 在许多基准测试中优于其他开源模型（如 Falcon 和 MPT）。\n",
+    "- LLaMA 2-Chat 提供了与 ChatGPT 类似的对话能力，适用于对话任务。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 应用场景**\n",
+    "\n",
+    "1. **研究**：\n",
+    "   - 开源权重适合学术研究，推动了对大语言模型的进一步探索。\n",
+    "\n",
+    "2. **对话系统**：\n",
+    "   - LLaMA 2-Chat 专为对话任务设计，适合开发智能客服、聊天机器人等应用。\n",
+    "\n",
+    "3. **生成任务**：\n",
+    "   - 支持文本生成、补全、摘要等任务。\n",
+    "\n",
+    "4. **微调与定制**：\n",
+    "   - 可以基于特定领域数据进行微调，如医学、法律、教育等领域的专用模型。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 开源与获取方式**\n",
+    "\n",
+    "#### **1. 开源**\n",
+    "- LLaMA 1：需要申请权限才能获得模型权重。\n",
+    "- LLaMA 2：更加开放，允许商业用途，模型和权重可以通过 Meta 的合作平台获取（如 Hugging Face 和 AWS）。\n",
+    "\n",
+    "#### **2. 下载与使用**\n",
+    "使用 Hugging Face 加载模型：\n",
+    "```python\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "model_name = \"meta-llama/Llama-2-7b-hf\"  # 替换为具体模型\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 使用模型生成文本\n",
+    "inputs = tokenizer(\"Hello, how are you?\", return_tensors=\"pt\")\n",
+    "outputs = model.generate(**inputs, max_length=50)\n",
+    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 总结**\n",
+    "\n",
+    "#### **优势**\n",
+    "- **高性能**：在多个基准任务上表现出色。\n",
+    "- **高效训练**：小参数模型能与大模型媲美。\n",
+    "- **开放性**：LLaMA 2 提供了较为开放的商用许可。\n",
+    "\n",
+    "#### **局限**\n",
+    "- 模型需要高质量数据和强大算力训练，对推理设备也有一定要求。\n",
+    "\n",
+    "LLaMA 系列以其高效和开放的特点，为大模型研究和应用带来了强大动力，是当前大语言模型生态的重要组成部分。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7fb0d648-f891-47b9-a644-af5263fa9718",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8b3c9ebb-213b-4dc4-a712-5a819fea3197",
+   "metadata": {},
+   "source": [
+    "**大模型的持续预训练**（Continual Pretraining of Large Models）是指在基础预训练模型（如 GPT、BERT 等）的基础上，通过引入新的数据或特定领域的数据继续进行预训练的过程。这一过程旨在让模型在特定场景或任务中表现更好，同时保留其通用能力。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 持续预训练的概念**\n",
+    "\n",
+    "持续预训练是一种在通用大模型的预训练基础上，进一步优化和适配模型的方法，主要包括以下两种场景：\n",
+    "1. **领域适配**：\n",
+    "   - 将预训练模型在特定领域的数据上继续训练，使其对该领域的语料理解更深刻，例如法律、医学、金融等领域。\n",
+    "2. **性能优化**：\n",
+    "   - 通过引入更多的通用数据或多样化的数据类型，扩展模型的通用能力，提高性能。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 持续预训练的目标**\n",
+    "\n",
+    "1. **提升领域性能**：\n",
+    "   - 在特定领域任务上，模型能够更好地理解特定领域的语言模式和知识。\n",
+    "   \n",
+    "2. **增强模型鲁棒性**：\n",
+    "   - 通过引入新的数据或增强数据多样性，使模型对未见数据表现更稳定。\n",
+    "\n",
+    "3. **优化资源利用**：\n",
+    "   - 通过复用已有的大模型权重，只需训练少量额外步骤，避免从零开始重新训练模型。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 持续预训练的步骤**\n",
+    "\n",
+    "#### **（1）数据准备**\n",
+    "- **领域数据**：针对特定领域（如医学、法律、科技）收集高质量语料。\n",
+    "- **新语料整合**：补充模型未见过的多样化语料。\n",
+    "- **数据清洗**：确保数据无噪声、语言风格一致。\n",
+    "\n",
+    "#### **（2）模型初始化**\n",
+    "- 使用现有的预训练模型作为初始权重，例如 Hugging Face 提供的 GPT-2 或 BERT 模型。\n",
+    "\n",
+    "#### **（3）训练设置**\n",
+    "- **超参数调整**：\n",
+    "  - 通常使用较小的学习率（例如 `1e-5` 或 `2e-5`）以避免破坏已有的知识。\n",
+    "- **训练策略**：\n",
+    "  - 冻结部分参数（如嵌入层或前几层）以保留通用能力，仅调整高层或新加入的部分。\n",
+    "\n",
+    "#### **（4）评估和验证**\n",
+    "- 使用领域任务的数据集对模型进行评估，验证其在目标任务中的改进效果。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 持续预训练的常见方法**\n",
+    "\n",
+    "#### **（1）全量持续预训练**\n",
+    "- 对整个模型的参数进行调整。\n",
+    "- **优点**：适合较大规模的新数据训练，能显著提升领域性能。\n",
+    "- **缺点**：计算资源需求大，可能导致模型过拟合。\n",
+    "\n",
+    "#### **（2）冻结部分参数**\n",
+    "- 冻结低层参数，仅微调高层。\n",
+    "- **优点**：保留通用知识，减少计算开销。\n",
+    "- **缺点**：对领域特定知识的适配可能不足。\n",
+    "\n",
+    "#### **（3）参数高效微调（PEFT）**\n",
+    "- 使用 PEFT 方法（如 LoRA、Adapter）进行预训练：\n",
+    "  - **LoRA**：通过低秩矩阵分解，微调部分关键模块。\n",
+    "  - **Adapter**：在 Transformer 层中插入小型适配模块。\n",
+    "- **优点**：显著减少需要更新的参数量。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 持续预训练的典型应用**\n",
+    "\n",
+    "1. **领域适配**\n",
+    "   - **医学**：将预训练模型在 PubMed 或生物医学数据集上进行持续预训练。\n",
+    "   - **法律**：使用法律文档进一步训练基础模型。\n",
+    "   - **金融**：通过金融新闻、报告语料提升模型在金融领域的表现。\n",
+    "\n",
+    "2. **多语言扩展**\n",
+    "   - 引入多语言语料，扩展模型的多语言能力。\n",
+    "\n",
+    "3. **数据更新**\n",
+    "   - 持续加入新数据（如时事新闻）以适配最新语言模式。\n",
+    "\n",
+    "4. **特殊任务优化**\n",
+    "   - 针对特定任务（如代码生成、对话）引入专用数据进行训练。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 实现持续预训练的代码示例**\n",
+    "\n",
+    "以下示例基于 Hugging Face 实现 GPT-2 的持续预训练：\n",
+    "\n",
+    "```python\n",
+    "from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# 1. 加载预训练模型和分词器\n",
+    "model_name = \"gpt2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 2. 加载新语料数据\n",
+    "dataset = load_dataset(\"text\", data_files={\"train\": \"domain_corpus.txt\"})\n",
+    "\n",
+    "# 3. 数据预处理\n",
+    "def tokenize_function(examples):\n",
+    "    return tokenizer(examples[\"text\"], truncation=True, max_length=1024, padding=\"max_length\")\n",
+    "\n",
+    "tokenized_dataset = dataset.map(tokenize_function, batched=True)\n",
+    "\n",
+    "# 4. 设置训练参数\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2_domain_adapted\",\n",
+    "    overwrite_output_dir=True,\n",
+    "    per_device_train_batch_size=4,\n",
+    "    num_train_epochs=3,\n",
+    "    learning_rate=5e-5,\n",
+    "    save_steps=500,\n",
+    "    save_total_limit=2,\n",
+    "    logging_dir=\"./logs\",\n",
+    "    evaluation_strategy=\"no\",  # 评估策略可以根据需要调整\n",
+    "    fp16=True,  # 混合精度训练\n",
+    ")\n",
+    "\n",
+    "# 5. 定义 Trainer 并启动训练\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_dataset[\"train\"],\n",
+    "    tokenizer=tokenizer,\n",
+    ")\n",
+    "\n",
+    "trainer.train()\n",
+    "\n",
+    "# 6. 保存模型\n",
+    "model.save_pretrained(\"./gpt2_domain_adapted\")\n",
+    "tokenizer.save_pretrained(\"./gpt2_domain_adapted\")\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 持续预训练的挑战**\n",
+    "\n",
+    "1. **灾难性遗忘**：\n",
+    "   - 持续预训练可能导致模型丧失之前学到的知识。\n",
+    "   - **解决方法**：使用少量原始数据进行联合训练。\n",
+    "\n",
+    "2. **计算资源需求**：\n",
+    "   - 需要大量显存和算力，特别是对于大规模模型和数据。\n",
+    "\n",
+    "3. **数据质量和多样性**：\n",
+    "   - 新引入的数据可能包含噪声，影响模型性能。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **8. 持续预训练的优势**\n",
+    "\n",
+    "- 提高特定领域或任务的性能。\n",
+    "- 更高效地利用已有模型权重，避免从头训练。\n",
+    "- 保留原始模型的通用能力，同时增强领域适应性。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "持续预训练是适配领域任务和提升模型性能的重要方法，通过引入新数据或优化模型训练策略，可以让大模型在特定场景中表现更优。配合参数高效微调方法（如 LoRA），还可显著降低计算开销，提升训练效率。这种技术在学术研究、工业应用和前沿领域（如法律、医学等）中均具有广泛价值。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca41ad33-18fb-44da-8f79-0380b5c9dcaa",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3038550c-cc92-45c9-8bb4-46c58688bfc5",
+   "metadata": {},
+   "source": [
+    "## 本节任务\n",
+    "本节任务是基于llama。训练一个能够处理dna和protein蛋白质数据的基础预训练大模型，数据为第一章中的预训练数据，包括英文数据。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b1bd33b8-2e05-4b59-9d8f-c48de194cfd6",
+   "metadata": {},
+   "source": [
+    "## 代码运行\n",
+    "\n",
+    "```\n",
+    "# 复制第一章训练数据,包括dna，protein，还有英文数据，添加英文数据是为了避免遗忘问题\n",
+    "mkdir train_data\n",
+    "cp ../01-data_env/data/*.txt train_data/\n",
+    "\n",
+    "#持续预训练\n",
+    "./run_pt.sh\n",
+    "\n",
+    "#合并模型\n",
+    "./merge_sft_model.sh\n",
+    "\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4960a36c-7529-4db8-b91d-df91245f79d9",
+   "metadata": {},
+   "source": [
+    "## 模型验证"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69b3e97f-a801-4264-a651-a854bcfba9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "339435d9-9379-4b30-ae8b-50feee1ba714",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-merge-hf\")\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d0f154bb-b1ab-4611-a14c-9b403043fd96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-merge-hf\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "792a9f78-1828-4695-9f6e-479a704ea7e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoConfig\n",
+    "# 加载配置\n",
+    "config = AutoConfig.from_pretrained('dnahlm-merge-hf')\n",
+    "config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49021c65-54bb-4a97-a96d-b030cc3dcd13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text='''GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
+    "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+    "The primary use of LLaMA is research on large language models, including'''\n",
+    "print(\"Test text:\\n\",text)\n",
+    "print(f\"Tokenized by DNA-LLaMA tokenizer:{tokenizer.tokenize(text)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ebf869c8-866d-4770-8f64-79d671f88663",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from transformers import pipeline\n",
+    "\n",
+    "model_id = \"dnahlm-merge-hf\"\n",
+    "\n",
+    "pipe = pipeline(\n",
+    "    \"text-generation\", \n",
+    "    model=model_id, \n",
+    "    #torch_dtype=torch.bfloat16, \n",
+    "    device_map=\"auto\",\n",
+    ")\n",
+    "\n",
+    "pipe(\"The key to life is\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40a22c70-f1c4-4cd5-a118-2f5db40790e6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe(\"GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCT\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aec95d0a-4269-4540-bf14-4ce157b9a194",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/.ipynb_checkpoints/7-llama-instruction-ft-checkpoint.ipynb

@@ -0,0 +1,624 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "963e9ae0-ac68-44be-8c7d-fb9842784362",
+   "metadata": {},
+   "source": [
+    "# 4.7 基于llama的基因大模型指令微调"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c844103d-4e27-41b9-9bf1-c6a577846ab6",
+   "metadata": {},
+   "source": [
+    "### **大模型的指令微调（Instruction Fine-Tuning）**\n",
+    "\n",
+    "指令微调是指通过对大语言模型（如 GPT、T5、LLaMA 等）进行微调，使其能够更好地理解和执行人类以指令形式表达的任务。这种技术是大模型适配实际应用和增强用户交互能力的关键手段。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 指令微调的核心概念**\n",
+    "\n",
+    "指令微调的目标是通过在包含指令的专用数据集上进行微调，让模型能够：\n",
+    "1. 理解用户的任务需求（以自然语言表达的指令形式）。\n",
+    "2. 根据指令内容生成符合预期的高质量响应。\n",
+    "3. 适应多任务场景，减少特定任务的单独训练需求。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 指令微调的关键特点**\n",
+    "\n",
+    "1. **多任务统一**：\n",
+    "   - 不需要针对每个任务单独微调，而是通过指令微调使模型能适应多种任务。\n",
+    "   \n",
+    "2. **自然语言交互**：\n",
+    "   - 用户可以用自然语言指令与模型交互，无需提供特定格式的输入。\n",
+    "\n",
+    "3. **泛化能力**：\n",
+    "   - 微调后的模型能够对未见过的任务产生合理的推断和响应。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 数据集的构建与使用**\n",
+    "\n",
+    "#### **（1）指令微调数据集的特点**\n",
+    "- 数据通常包含以下三部分：\n",
+    "  1. **指令（Instruction）**：任务描述或问题，例如“将以下文本翻译为法语”。\n",
+    "  2. **输入（Input）**：任务相关的上下文或数据，可以为空。\n",
+    "  3. **输出（Output）**：模型期望生成的结果。\n",
+    "\n",
+    "#### **（2）常用指令微调数据集**\n",
+    "- **FLAN**：包含多个 NLP 任务的指令数据集，用于 T5 等模型的微调。\n",
+    "- **OpenAI 提供的指令数据**：如 GPT 系列的 ChatGPT 调优数据集。\n",
+    "- **InstructGPT 数据**：通过人类标注的多任务指令数据，用于模型优化。\n",
+    "- **Self-Instruct**：通过模型自生成指令和回答，进一步扩展训练数据。\n",
+    "\n",
+    "#### **（3）构建自己的数据集**\n",
+    "- 如果需要特定领域的指令微调，可以自行构建数据集：\n",
+    "  - 收集任务需求和示例。\n",
+    "  - 设计多样化的指令。\n",
+    "  - 使用专家标注或模型辅助生成高质量答案。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 微调的步骤**\n",
+    "\n",
+    "#### **（1）加载基础模型**\n",
+    "从 Hugging Face 或其他框架加载预训练的大语言模型，例如 GPT-2、T5、LLaMA。\n",
+    "\n",
+    "#### **（2）准备数据集**\n",
+    "将指令微调数据集格式化为：\n",
+    "```python\n",
+    "{\n",
+    "    \"instruction\": \"Translate the following text to French\",\n",
+    "    \"input\": \"Hello, how are you?\",\n",
+    "    \"output\": \"Bonjour, comment ça va?\"\n",
+    "}\n",
+    "```\n",
+    "\n",
+    "#### **（3）定义微调方法**\n",
+    "使用 `Trainer` 或分布式框架（如 DeepSpeed、Accelerate）进行微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 示例代码：指令微调实现**\n",
+    "\n",
+    "以下是基于 Hugging Face 的指令微调代码示例：\n",
+    "\n",
+    "```python\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# 1. 加载预训练模型和分词器\n",
+    "model_name = \"gpt2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 2. 加载指令微调数据集\n",
+    "# 数据格式应包含 instruction, input, output 字段\n",
+    "dataset = load_dataset(\"path/to/instruction_dataset\")\n",
+    "\n",
+    "# 3. 数据预处理\n",
+    "def preprocess_function(example):\n",
+    "    # 将指令和输入拼接成完整的提示\n",
+    "    prompt = example[\"instruction\"]\n",
+    "    if example[\"input\"]:\n",
+    "        prompt += f\"\\n{example['input']}\"\n",
+    "    labels = example[\"output\"]\n",
+    "    tokenized = tokenizer(prompt, truncation=True, max_length=512, padding=\"max_length\")\n",
+    "    with tokenizer.as_target_tokenizer():\n",
+    "        tokenized_labels = tokenizer(labels, truncation=True, max_length=512, padding=\"max_length\")\n",
+    "    tokenized[\"labels\"] = tokenized_labels[\"input_ids\"]\n",
+    "    return tokenized\n",
+    "\n",
+    "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
+    "\n",
+    "# 4. 设置训练参数\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./instruction_finetuned_model\",\n",
+    "    per_device_train_batch_size=4,\n",
+    "    num_train_epochs=3,\n",
+    "    evaluation_strategy=\"epoch\",\n",
+    "    save_strategy=\"epoch\",\n",
+    "    learning_rate=5e-5,\n",
+    "    weight_decay=0.01,\n",
+    "    logging_dir=\"./logs\",\n",
+    "    fp16=True,\n",
+    ")\n",
+    "\n",
+    "# 5. 定义 Trainer\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_datasets[\"train\"],\n",
+    "    eval_dataset=tokenized_datasets[\"test\"],\n",
+    "    tokenizer=tokenizer,\n",
+    ")\n",
+    "\n",
+    "# 6. 开始训练\n",
+    "trainer.train()\n",
+    "\n",
+    "# 7. 保存模型\n",
+    "model.save_pretrained(\"./instruction_finetuned_model\")\n",
+    "tokenizer.save_pretrained(\"./instruction_finetuned_model\")\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 指令微调的挑战**\n",
+    "\n",
+    "1. **数据质量**：\n",
+    "   - 低质量或噪声数据可能导致模型生成结果不符合指令。\n",
+    "\n",
+    "2. **指令覆盖范围**：\n",
+    "   - 数据集指令种类不足会限制模型的泛化能力。\n",
+    "\n",
+    "3. **计算资源需求**：\n",
+    "   - 大模型的微调需要高性能 GPU 和大容量存储。\n",
+    "\n",
+    "4. **灾难性遗忘**：\n",
+    "   - 微调过程中可能导致模型丧失部分原始能力。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 指令微调的应用场景**\n",
+    "\n",
+    "1. **多任务问答**：\n",
+    "   - 适配多任务场景，支持翻译、总结、推理等功能。\n",
+    "\n",
+    "2. **特定领域优化**：\n",
+    "   - 在法律、医疗等特定领域的任务指令上进行微调。\n",
+    "\n",
+    "3. **用户交互优化**：\n",
+    "   - 提升模型对自然语言指令的理解和响应能力。\n",
+    "\n",
+    "4. **开放式对话生成**：\n",
+    "   - 优化模型在对话场景下的表现，例如 ChatGPT 的微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "指令微调通过在特定格式的数据集上进一步训练大模型，使其能够更好地理解和执行用户的自然语言指令。这种方法适合多任务场景，并能提升模型的交互能力和领域适应性。借助高质量的指令数据集和高效的微调技术，大模型在实际应用中的表现可以得到显著提升。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7be8b814-42f6-4fb6-bf4b-ae23292030f6",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9bed0ae-337d-49af-85f0-c8e6263d78db",
+   "metadata": {},
+   "source": [
+    "**大模型的持续预训练**和**指令微调**是两种针对大模型的后续优化策略，虽然它们的目标都是提升模型性能，但在应用场景、方法和效果等方面有明显区别。以下是它们的对比分析：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 概念与目标**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **定义**               | 在通用预训练模型上，使用新的大规模语料（通用或领域特定数据）进行进一步预训练。                      | 在包含指令任务的数据集上对大模型进行微调，以提升模型对人类指令的理解和执行能力。                              |\n",
+    "| **目标**               | 提升模型的通用能力或适应特定领域的语言理解与生成能力。                                              | 提高模型对多任务指令的泛化能力，让模型更好地理解和执行自然语言表达的具体任务。                                |\n",
+    "| **典型应用**           | 领域适配（医学、法律、金融）、性能优化、跨语言适配等。                                              | 多任务问答、开放式对话生成、翻译、推理等需要用户直接交互的场景。                                              |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 数据使用**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **数据类型**           | 通用语料（如新闻、社交媒体文本）或领域特定语料（如 PubMed、法律文档、金融报告）。                    | 任务指令数据集，包括指令（Instruction）、输入（Input）和输出（Output）。                                   |\n",
+    "| **数据构建**           | 通常需要清洗和去重大规模语料数据，避免与原始预训练数据重叠。                                         | 通常由人工标注或模型生成的指令数据构成，例如 FLAN、InstructGPT 数据集。                                     |\n",
+    "| **多样性要求**         | 数据应覆盖尽可能广的领域或目标领域的多种场景，以提升模型在这些场景的表现。                                | 数据需要覆盖多种任务类型（如翻译、分类、摘要）和丰富的指令表达形式，以提高模型对多任务的适配能力。                 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 方法与技术**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **主要技术**           | 继续使用自监督学习目标（如语言建模、掩码预测）进行训练。                                            | 使用监督学习，通常以任务输入和目标输出对为数据，通过微调适配特定任务需求。                                  |\n",
+    "| **模型调整**           | - 可选择全量参数更新或冻结部分参数。<br>- 可结合参数高效微调技术（如 LoRA、Adapter）。                  | - 通常使用监督训练方式，可能结合参数高效微调技术（如 LoRA）。                                               |\n",
+    "| **学习率**             | 通常使用较小的学习率（如 `1e-5` 或更小），以防止破坏原始权重。                                         | 同样使用较小的学习率，但任务指令微调可能需要更高的关注任务特定的标签对准。                                     |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 模型能力与效果**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **提升的能力**         | - 对领域特定语言模式和知识的适配性提升显著。<br>- 对未见过的通用场景生成能力增强（扩展模型知识广度）。       | - 显著提升模型对指令理解的能力，尤其是自然语言表达的任务需求。<br>- 对多任务和零样本任务的泛化能力有较大提升。  |\n",
+    "| **局限性**             | - 对具体任务的直接适配能力较弱，可能需要额外的任务微调。<br>- 数据选择不当可能导致灾难性遗忘。                 | - 依赖高质量的指令数据集，数据质量不高会导致模型生成结果不稳定。<br>- 对通用能力的提升有限。                    |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 应用场景与示例**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **典型应用场景**       | - 医学文献总结（通过 PubMed 语料持续预训练）。<br>- 法律条文分析（通过法律文档进一步训练）。<br>- 增强多语言生成能力（跨语言语料）。 | - ChatGPT 的多任务对话生成。<br>- 翻译、摘要、问答等用户交互任务的泛化处理。                                 |\n",
+    "| **实际示例**           | - BioBERT：在 BERT 基础上使用生物医学语料持续预训练的模型。<br>- FinBERT：针对��融领域持续预训练的语言模型。 | - InstructGPT：在 GPT-3 基础上进行指令微调，用于多任务用户交互。<br>- FLAN-T5：通过 FLAN 数据集进行指令微调。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 持续预训练与指令微调的结合**\n",
+    "\n",
+    "持续预训练和指令微调可以结合使用，形成一个从领域适配到任务适配的完整流程：\n",
+    "1. **持续预训练**：\n",
+    "   - 先在领域特定数据（如医学、法律、金融语料）上进行持续预训练，获取领域知识。\n",
+    "2. **指令微调**：\n",
+    "   - 再利用多任务指令数据集对模型微调，使其能够高效执行领域内的多样化任务。\n",
+    "\n",
+    "这种结合方式特别适用于需要领域知识和任务适配的场景，例如医学问答系统或金融文本分析。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "| **维度**               | **持续预训练**                      | **指令微调**                     |\n",
+    "|------------------------|-------------------------------------|----------------------------------|\n",
+    "| **目标**               | 增强通用能力或适配特定领域。          | 提升对任务指令的理解和执行能力。    |\n",
+    "| **数据集**             | 通用或领域语料。                    | 指令数据集，包含输入和输出对。      |\n",
+    "| **方法**               | 自监督学习，扩展语言建模能力。         | 监督学习，强化任务适配能力。        |\n",
+    "| **适用场景**           | 领域特定任务（如医学、法律）。         | 多任务交互（如问答、对话生成）。     |\n",
+    "| **局限性**             | 对具体任务适配较弱。                 | 通用能力提升有限，依赖数据质量。     |\n",
+    "\n",
+    "两者各有侧重，且在许多场景下可以结合使用，形成一个强大的任务和领域适配框架。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f97a705a-b946-4dc1-a173-a9df033d6f2b",
+   "metadata": {},
+   "source": [
+    "## 本节任务\n",
+    "本节任务是基于上一节预训练的llama生物大模型。对一些生物学任务进行微调，包含了多个不同类型的分类问题和多序列交换问题。具体可见sft_data下的数据。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9782db62-95bd-40a6-9759-966b9a0b362e",
+   "metadata": {},
+   "source": [
+    "## 代码运行\n",
+    "\n",
+    "```\n",
+    "\n",
+    "#微调\n",
+    "./run_sft.sh\n",
+    "\n",
+    "#合并模型\n",
+    "./merge_sft_model.sh\n",
+    "\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "182b82c4-d484-4c15-a600-03c3b51367ec",
+   "metadata": {},
+   "source": [
+    "## 模型验证"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5aa3d240-44e1-4811-8f61-d6ff2500a798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
+   "metadata": {},
+   "source": [
+    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "054a2956-9045-4ad5-a878-1bfc84ad4ed8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63c8bf16-9576-41bc-b27c-c92ba4289cf4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
+    "dna_ft_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95928da3-ca64-4a17-80f4-945da395702c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a3e65bcd-85ce-4261-8ba6-7665c4ec60e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #dnagpt/dnahlm-llama-7b-sft-v0\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d3fe49b-f48f-42b2-bc97-028e443111e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c54df9fe-86c4-4963-b313-b438894bf9dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#构建提示词\n",
+    "def format_input(entry):\n",
+    "    instruction_text = (\n",
+    "        f\"Below is an instruction that describes a task. \"\n",
+    "        f\"Write a response that appropriately completes the request.\"\n",
+    "        f\"\\n\\n### Instruction:\\n{entry['instruction']}\"\n",
+    "    )\n",
+    "\n",
+    "    input_text = f\"\\n\\n### Input:\\n{entry['input']}\" if entry[\"input\"] else \"\"\n",
+    "\n",
+    "    return instruction_text + input_text + \"\\n\\n### Response:\\n\"\n",
+    "\n",
+    "#构建提示词\n",
+    "def build_prompt(entry):\n",
+    "\n",
+    "    input_data = format_input(entry)\n",
+    "\n",
+    "    desired_response = entry['output']\n",
+    "\n",
+    "    return input_data + desired_response\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee540cfb-1f6e-4e02-a3bc-c814e43685cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "example = data[\"test\"][0]\n",
+    "example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ee35528-7b3f-4e60-b88b-1bc3e950012b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = build_prompt(example)\n",
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8aa6f38f-3bcc-4566-8a66-a541db91e031",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.tokenize(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11875339-4901-4912-86e5-afe8c74921d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=1000):\n",
+    "  # Tokenize\n",
+    "  input_ids = tokenizer.encode(\n",
+    "          text,\n",
+    "          return_tensors=\"pt\",\n",
+    "          truncation=True,\n",
+    "          max_length=max_input_tokens\n",
+    "          # return_attention_mask=True,\n",
+    "  )\n",
+    "\n",
+    "  # Generate\n",
+    "  device = model.device\n",
+    "  generated_tokens_with_prompt = model.generate(\n",
+    "    input_ids=input_ids.to(device),\n",
+    "    #max_length=max_output_tokens,\n",
+    "    max_new_tokens=8,\n",
+    "    temperature=0.01  # 控制生成的多样性\n",
+    "  )\n",
+    "\n",
+    "  # Decode\n",
+    "  generated_text_with_prompt = tokenizer.decode(generated_tokens_with_prompt[0], skip_special_tokens=True)\n",
+    "  generated_text_answer = generated_text_with_prompt[len(text):]\n",
+    "\n",
+    "\n",
+    "  return generated_text_answer\n",
+    "\n",
+    "# 如果需要进一步清理\n",
+    "def clean_generated_text(text):\n",
+    "    # 去除 'Ġ' 符号并替换为空格\n",
+    "    text = text.replace('Ġ', ' ')\n",
+    "    # 去除多余的空格\n",
+    "    text = ' '.join(text.split())\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b02644a-8b24-45aa-b22d-0f7ce2270dd9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_text = format_input(data[\"test\"][0])\n",
+    "\n",
+    "print(\"input (test):\", input_text)\n",
+    "\n",
+    "print(\"real answer:\", data[\"test\"][0][\"output\"])\n",
+    "\n",
+    "print(\"--------------------------\\n\")\n",
+    "\n",
+    "print(\"model's answer: \\n\")\n",
+    "print(inference(input_text, model, tokenizer))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e2df1569-7f70-46ee-b93f-cbd879e32e14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_data = data[\"test\"].shuffle(seed=199).select(range(100))\n",
+    "\n",
+    "data_list = []\n",
+    "\n",
+    "for entry in test_data:\n",
+    "    input_text = format_input(entry)\n",
+    "    #print(input_text)\n",
+    "    response_text = inference(input_text, model, tokenizer)\n",
+    "    #print(response_text)\n",
+    "    data = {\n",
+    "        \"instruction\":entry[\"instruction\"],\n",
+    "         \"input\":entry[\"input\"],\n",
+    "         \"output\":entry[\"output\"],\n",
+    "        \"model_response\":response_text\n",
+    "    }\n",
+    "\n",
+    "    data_list.append(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c6e47cb-1b64-4690-a51d-f1816b82f15f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 定义输出文件路径\n",
+    "output_file = 'llama-sft-2.json'\n",
+    "\n",
+    "# 将 Dataset 对象导出为 JSON 文件\n",
+    "# test_data.to_json(output_file)\n",
+    "with open(output_file, \"w\") as file:\n",
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68831e19-5a99-46d8-9f40-e8bf6957dbfc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "\n",
+    "\n",
+    "with open(output_file, \"r\") as file:\n",
+    "    test_data = json.load(file)\n",
+    "\n",
+    "all_num = len(test_data)\n",
+    "right_sum = 0\n",
+    "same_sum = 0\n",
+    "for item in test_data:\n",
+    "    output = item[\"output\"]\n",
+    "    #output = \" \".join(tokenizer.tokenize(output))\n",
+    "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
+    "    if model_response == output: #same it\n",
+    "        same_sum = same_sum + 1\n",
+    "        \n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
+    "\n",
+    "\n",
+    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/.ipynb_checkpoints/llama_sft_test-checkpoint.ipynb

@@ -0,0 +1,1627 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "81a2413e-8629-4016-aace-17d2f757f726",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "https://hf-mirror.com\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "# 设置环境变量\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
+    "\n",
+    "# 打印环境变量以确认设置成功\n",
+    "print(os.environ.get('HF_ENDPOINT'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "89e2d33a-6d84-4ef3-b44e-daa57ac81e58",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-11-24 11:21:51.020375: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
+      "2024-11-24 11:21:51.036615: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
+      "2024-11-24 11:21:51.053557: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
+      "2024-11-24 11:21:51.058466: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
+      "2024-11-24 11:21:51.071840: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX512_FP16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "2024-11-24 11:21:51.923693: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "68fc5c44-b444-402e-aaf2-0ba4e2000e42",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 19839\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
+    "dna_ft_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4ab4fd3e-5b59-470e-9b46-f0ffd7b9d1aa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 1983\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 17856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "85ca97f5-6864-4d6f-944a-182ed1fa2f00",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #dnagpt/dnahlm-llama-7b-sft-v0\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e904c0b2-bf21-4036-b510-8e57177c1767",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "99ce92d0373a498d929bed42f770ed16",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(61973, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=61973, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "5b361c5c-c43f-4ed9-a5c7-c72403cd7a0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#构建提示词\n",
+    "def format_input(entry):\n",
+    "    instruction_text = (\n",
+    "        f\"Below is an instruction that describes a task. \"\n",
+    "        f\"Write a response that appropriately completes the request.\"\n",
+    "        f\"\\n\\n### Instruction:\\n{entry['instruction']}\"\n",
+    "    )\n",
+    "\n",
+    "    input_text = f\"\\n\\n### Input:\\n{entry['input']}\" if entry[\"input\"] else \"\"\n",
+    "\n",
+    "    return instruction_text + input_text + \"\\n\\n### Response:\\n\"\n",
+    "\n",
+    "#构建提示词\n",
+    "def build_prompt(entry):\n",
+    "\n",
+    "    input_data = format_input(entry)\n",
+    "\n",
+    "    desired_response = entry['output']\n",
+    "\n",
+    "    return input_data + desired_response\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "ed031a26-d79e-4f50-85d1-169ebd409c6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "example = data[\"test\"][0]\n",
+    "example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "31bd4bb5-86a6-4046-b510-492b0548323b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
+   "source": [
+    "prompt = build_prompt(example)\n",
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "ed0b5b8b-916c-499b-a6da-f1124b9add1c",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['▁Below',\n",
+       " '▁is',\n",
+       " '▁an',\n",
+       " '▁instruction',\n",
+       " '▁that',\n",
+       " '▁describes',\n",
+       " '▁a',\n",
+       " '▁task',\n",
+       " '.',\n",
+       " '▁Write',\n",
+       " '▁a',\n",
+       " '▁response',\n",
+       " '▁that',\n",
+       " '▁appropri',\n",
+       " 'ately',\n",
+       " '▁comple',\n",
+       " 'tes',\n",
+       " '▁the',\n",
+       " '▁request',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Inst',\n",
+       " 'ruction',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'Det',\n",
+       " 'erm',\n",
+       " 'ine',\n",
+       " '▁core',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '▁detection',\n",
+       " '▁of',\n",
+       " '▁following',\n",
+       " '▁d',\n",
+       " 'na',\n",
+       " '▁sequence',\n",
+       " ',',\n",
+       " '▁The',\n",
+       " '▁result',\n",
+       " '▁will',\n",
+       " '▁be',\n",
+       " '▁one',\n",
+       " '▁of',\n",
+       " '▁the',\n",
+       " '▁following',\n",
+       " ':',\n",
+       " '▁Non',\n",
+       " '-',\n",
+       " 'prom',\n",
+       " 'oter',\n",
+       " ',',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Input',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'CCGTG',\n",
+       " 'C',\n",
+       " 'GAC',\n",
+       " 'CGGAA',\n",
+       " 'GTG',\n",
+       " 'GGGC',\n",
+       " 'GGC',\n",
+       " 'GAC',\n",
+       " 'CCCGGAA',\n",
+       " 'GTCC',\n",
+       " 'CCGCC',\n",
+       " 'GGGTG',\n",
+       " 'CA',\n",
+       " 'GCT',\n",
+       " 'TG',\n",
+       " 'GTC',\n",
+       " 'GGT',\n",
+       " 'TC',\n",
+       " 'GATCGCC',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Response',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'prom',\n",
+       " 'oter']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.tokenize(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "f0449aee-1ac6-4db5-873f-afdfb0fc9691",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=1000):\n",
+    "  # Tokenize\n",
+    "  input_ids = tokenizer.encode(\n",
+    "          text,\n",
+    "          return_tensors=\"pt\",\n",
+    "          truncation=True,\n",
+    "          max_length=max_input_tokens\n",
+    "          # return_attention_mask=True,\n",
+    "  )\n",
+    "\n",
+    "  # Generate\n",
+    "  device = model.device\n",
+    "  generated_tokens_with_prompt = model.generate(\n",
+    "    input_ids=input_ids.to(device),\n",
+    "    #max_length=max_output_tokens,\n",
+    "    max_new_tokens=8,\n",
+    "    temperature=0.01  # 控制生成的多样性\n",
+    "  )\n",
+    "\n",
+    "  # Decode\n",
+    "  generated_text_with_prompt = tokenizer.decode(generated_tokens_with_prompt[0], skip_special_tokens=True)\n",
+    "  generated_text_answer = generated_text_with_prompt[len(text):]\n",
+    "\n",
+    "\n",
+    "  return generated_text_answer\n",
+    "\n",
+    "# 如果需要进一步清理\n",
+    "def clean_generated_text(text):\n",
+    "    # 去除 'Ġ' 符号并替换为空格\n",
+    "    text = text.replace('Ġ', ' ')\n",
+    "    # 去除多余的空格\n",
+    "    text = ' '.join(text.split())\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "e9041426-eb59-4314-82dd-7b6d6d477783",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "real answer: promoter\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/configuration_utils.py:601: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.01` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " Non-promoter\n"
+     ]
+    }
+   ],
+   "source": [
+    "input_text = format_input(data[\"test\"][0])\n",
+    "\n",
+    "print(\"input (test):\", input_text)\n",
+    "\n",
+    "print(\"real answer:\", data[\"test\"][0][\"output\"])\n",
+    "\n",
+    "print(\"--------------------------\\n\")\n",
+    "\n",
+    "print(\"model's answer: \\n\")\n",
+    "print(inference(input_text, model, tokenizer))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "d1489173-84af-4c8e-b66b-0cdbe42c7ea7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_data = data[\"test\"].shuffle(seed=199).select(range(1000))\n",
+    "\n",
+    "data_list = []\n",
+    "\n",
+    "for entry in test_data:\n",
+    "    input_text = format_input(entry)\n",
+    "    #print(input_text)\n",
+    "    response_text = inference(input_text, model, tokenizer)\n",
+    "    #print(response_text)\n",
+    "    data = {\n",
+    "        \"instruction\":entry[\"instruction\"],\n",
+    "         \"input\":entry[\"input\"],\n",
+    "         \"output\":entry[\"output\"],\n",
+    "        \"model_response\":response_text\n",
+    "    }\n",
+    "\n",
+    "    data_list.append(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "39275fe6-ac3b-4558-9f4c-2853a41d48c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 定义输出文件路径\n",
+    "output_file = 'llama-sft-2.json'\n",
+    "\n",
+    "# 将 Dataset 对象导出为 JSON 文件\n",
+    "# test_data.to_json(output_file)\n",
+    "with open(output_file, \"w\") as file:\n",
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "7ffaba65-a270-4433-b234-932f5e288f7c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'▁prom oter'"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\" \".join(tokenizer.tokenize(\"promoter\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a7e373a4-6857-4874-b2da-58da2928925d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites |||||||||||| Courses\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites |||||||||||| Court\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites |||||||||||| Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites |||||||||||| Coursing\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Acceptor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites |||||||||||| Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Donor Sites |||||||||||| D Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| C promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-Splice Sites |||||||||||| Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites |||||||||||| Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites |||||||||||| Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Acceptor Sites\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "presicion 0.739 same 0.253\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "\n",
+    "\n",
+    "with open(output_file, \"r\") as file:\n",
+    "    test_data = json.load(file)\n",
+    "\n",
+    "all_num = len(test_data)\n",
+    "right_sum = 0\n",
+    "same_sum = 0\n",
+    "for item in test_data:\n",
+    "    output = item[\"output\"]\n",
+    "    #output = \" \".join(tokenizer.tokenize(output))\n",
+    "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
+    "    if model_response == output: #same it\n",
+    "        same_sum = same_sum + 1\n",
+    "        \n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
+    "\n",
+    "\n",
+    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "294d46f3-2f5b-4e55-ae41-081d5195f5e2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/.ipynb_checkpoints/merge_pt_model-checkpoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+python merge_llama_with_dna_lora.py \
+    --base_model llama-7b-hf \
+    --lora_model dnahlm_llama_7b/pt_lora_model \
+    --output_type huggingface \
+    --output_dir dnahlm-merge-hf

04-gene-sft/.ipynb_checkpoints/merge_sft_model-checkpoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+python merge_llama_with_dna_lora.py \
+    --base_model dnahlm-merge-hf \
+    --lora_model dnahlm-llama7b-sft/sft_lora_model \
+    --output_type huggingface \
+    --output_dir dnahlm-llama-7b-sft-v0

04-gene-sft/.ipynb_checkpoints/run_clm_pt_with_peft-checkpoint.py

@@ -0,0 +1,637 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+
+import logging
+import numpy as np
+import math
+import os
+import sys
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Optional, List, Dict, Any, Mapping
+from pathlib import Path
+import datasets
+import torch
+from datasets import load_dataset, concatenate_datasets
+
+import transformers
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    LlamaForCausalLM,
+    LlamaTokenizer,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    is_torch_tpu_available,
+    set_seed,
+)
+from transformers.testing_utils import CaptureLogger
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import send_example_telemetry
+from transformers.utils.versions import require_version
+
+from sklearn.metrics import accuracy_score
+from peft import LoraConfig, TaskType, get_peft_model, PeftModel, get_peft_model_state_dict
+from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+
+
+class SavePeftModelCallback(transformers.TrainerCallback):
+    def save_model(self, args, state, kwargs):
+        if state.best_model_checkpoint is not None:
+            checkpoint_folder = os.path.join(state.best_model_checkpoint, "pt_lora_model")
+        else:
+            checkpoint_folder = os.path.join(args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}")
+
+        peft_model_path = os.path.join(checkpoint_folder, "pt_lora_model")
+        kwargs["model"].save_pretrained(peft_model_path)
+
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
+    def on_save(self, args, state, control, **kwargs):
+        self.save_model(args, state, kwargs)
+        return control
+
+    def on_train_end(self, args, state, control, **kwargs):
+        peft_model_path = os.path.join(args.output_dir, "pt_lora_model")
+        kwargs["model"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
+
+def accuracy(predictions, references, normalize=True, sample_weight=None):
+        return {
+            "accuracy": float(
+                accuracy_score(references, predictions, normalize=normalize, sample_weight=sample_weight)
+            )
+        }
+
+
+def compute_metrics(eval_preds):
+    preds, labels = eval_preds
+    # preds have the same shape as the labels, after the argmax(-1) has been calculated
+    # by preprocess_logits_for_metrics but we need to shift the labels
+    labels = labels[:, 1:].reshape(-1)
+    preds = preds[:, :-1].reshape(-1)
+    return accuracy(predictions=preds, references=labels)
+
+
+def preprocess_logits_for_metrics(logits, labels):
+    if isinstance(logits, tuple):
+        # Depending on the model and config, logits may contain extra tensors,
+        # like past_key_values, but logits always come first
+        logits = logits[0]
+    return logits.argmax(dim=-1)
+
+
+def fault_tolerance_data_collator(features: List) -> Dict[str, Any]:
+    if not isinstance(features[0], Mapping):
+        features = [vars(f) for f in features]
+    first = features[0]
+    batch = {}
+
+    # Special handling for labels.
+    # Ensure that tensor is created with the correct type
+    # (it should be automatically the case, but let's make sure of it.)
+    if "label" in first and first["label"] is not None:
+        label = first["label"].item() if isinstance(first["label"], torch.Tensor) else first["label"]
+        dtype = torch.long if isinstance(label, int) else torch.float
+        batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype)
+    elif "label_ids" in first and first["label_ids"] is not None:
+        if isinstance(first["label_ids"], torch.Tensor):
+            batch["labels"] = torch.stack([f["label_ids"] for f in features])
+        else:
+            dtype = torch.long if isinstance(first["label_ids"][0], int) else torch.float
+            batch["labels"] = torch.tensor([f["label_ids"] for f in features], dtype=dtype)
+
+    # Handling of all other possible keys.
+    # Again, we will use the first element to figure out which key/values are not None for this model.
+
+    try:
+        for k, v in first.items():
+            if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
+                if isinstance(v, torch.Tensor):
+                    batch[k] = torch.stack([f[k] for f in features])
+                elif isinstance(v, np.ndarray):
+                    batch[k] = torch.tensor(np.stack([f[k] for f in features]))
+                else:
+                    batch[k] = torch.tensor([f[k] for f in features])
+    except ValueError: # quick fix by simply take the first example
+        for k, v in first.items():
+            if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
+                if isinstance(v, torch.Tensor):
+                    batch[k] = torch.stack([features[0][k]] * len(features))
+                elif isinstance(v, np.ndarray):
+                    batch[k] = torch.tensor(np.stack([features[0][k]] * len(features)))
+                else:
+                    batch[k] = torch.tensor([features[0][k]] * len(features))
+
+    return batch
+
+
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The tokenizer for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
+    )
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
+                "with private models)."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    dataset_dir: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
+    block_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional input sequence length after tokenization. "
+                "The training dataset will be truncated in block of this size for training. "
+                "Default to the model max input length for single sentence inputs (take into account special tokens)."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[float] = field(
+        default=0.05,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+    data_cache_dir: Optional[str] = field(default="./", metadata={"help": "The datasets processed stored"})
+
+    def __post_init__(self):
+        if self.streaming:
+            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")
+
+
+@dataclass
+class MyTrainingArguments(TrainingArguments):
+    trainable : Optional[str] = field(default="q_proj,v_proj")
+    lora_rank : Optional[int] = field(default=8)
+    lora_dropout : Optional[float] = field(default=0.1)
+    lora_alpha : Optional[float] = field(default=32.)
+    modules_to_save : Optional[str] = field(default=None)
+    debug_mode : Optional[bool] = field(default=False)
+    peft_path : Optional[str] = field(default=None)
+
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, MyTrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,  # if training_args.local_rank in [-1, 0] else logging.WARN,
+        handlers=[logging.StreamHandler(sys.stdout)],)
+
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+    # transformers.tokenization_utils.logging.set_verbosity_warning()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.tokenizer_name_or_path:
+        tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
+    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
+
+    def tokenize_function(examples):
+        with CaptureLogger(tok_logger) as cl:
+            output = tokenizer(examples["text"])
+        # clm input could be much much longer than block_size
+        if "Token indices sequence length is longer than the" in cl.out:
+            tok_logger.warning(
+                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
+                " before being passed to the model."
+            )
+        return output
+    if data_args.block_size is None:
+        block_size = tokenizer.model_max_length
+        if block_size > 1024:
+            logger.warning(
+                "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
+                " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
+                " override this default with `--block_size xxx`."
+            )
+            block_size = 1024
+    else:
+        if data_args.block_size > tokenizer.model_max_length:
+            logger.warning(
+                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model"
+                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
+            )
+        block_size = min(data_args.block_size, tokenizer.model_max_length)
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
+        # customize this part to your needs.
+        if total_length >= block_size:
+            total_length = (total_length // block_size) * block_size
+        # Split by chunks of max_len.
+        result = {
+            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+            for k, t in concatenated_examples.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+    with training_args.main_process_first(desc="dataset map tokenization and grouping"):
+        lm_datasets = []
+        path = Path(data_args.dataset_dir)
+        files = [file.name for file in path.glob("*.txt")]
+        if training_args.debug_mode is True:
+            files = [files[0]]
+        for idx, file in enumerate(files):
+            data_file = os.path.join(path, file)
+            filename = ''.join(file.split(".")[:-1])
+            cache_path = os.path.join(data_args.data_cache_dir, filename)
+            os.makedirs(cache_path, exist_ok=True)
+            try:
+                processed_dataset = datasets.load_from_disk(cache_path, keep_in_memory=False)
+                logger.info(f'training datasets-{filename} has been loaded from disk')
+            except Exception:
+                cache_dir = os.path.join(data_args.data_cache_dir, filename+"_text")
+                os.makedirs(cache_dir, exist_ok=True)
+                raw_dataset = load_dataset("text", data_files=data_file, cache_dir=cache_dir, keep_in_memory=False)
+                logger.info(f"{file} has been loaded")
+                tokenized_dataset = raw_dataset.map(
+                    tokenize_function,
+                    batched=True,
+                    num_proc=data_args.preprocessing_num_workers,
+                    remove_columns="text",
+                    load_from_cache_file=True,
+                    keep_in_memory=False,
+                    cache_file_names = {k: os.path.join(cache_dir, 'tokenized.arrow') for k in raw_dataset},
+                    desc="Running tokenizer on dataset",
+                )
+                grouped_datasets = tokenized_dataset.map(
+                    group_texts,
+                    batched=True,
+                    num_proc=data_args.preprocessing_num_workers,
+                    load_from_cache_file=True,
+                    keep_in_memory=False,
+                    cache_file_names = {k: os.path.join(cache_dir, 'grouped.arrow') for k in tokenized_dataset},
+                    desc=f"Grouping texts in chunks of {block_size}",
+                )
+                processed_dataset = grouped_datasets
+                processed_dataset.save_to_disk(cache_path)
+            if idx == 0:
+                lm_datasets = processed_dataset['train']
+            else:
+                assert lm_datasets.features.type == processed_dataset["train"].features.type
+                lm_datasets = concatenate_datasets([lm_datasets, processed_dataset["train"]])
+
+        lm_datasets = lm_datasets.train_test_split(test_size = data_args.validation_split_percentage)
+
+    if training_args.do_train:
+        train_dataset = lm_datasets['train']
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+        logger.info(f"Num train_samples  {len(train_dataset)}")
+        logger.info("training example:")
+        logger.info(tokenizer.decode(train_dataset[0]['input_ids']))
+    if training_args.do_eval:
+        eval_dataset = lm_datasets["test"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+        logger.info(f"Num eval_samples  {len(eval_dataset)}")
+        logger.info("training example:")
+        logger.info(tokenizer.decode(eval_dataset[0]['input_ids']))
+
+
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = LlamaForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            use_auth_token=True if model_args.use_auth_token else None,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=True
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    model_vocab_size = model.get_output_embeddings().weight.size(0)
+
+    model.resize_token_embeddings(len(tokenizer))
+    if training_args.peft_path is not None:
+        logger.info("Peft from pre-trained model")
+        model = PeftModel.from_pretrained(model, training_args.peft_path)
+    else:
+        logger.info("Init new peft model")
+        target_modules = training_args.trainable.split(',')
+        modules_to_save = training_args.modules_to_save
+        if modules_to_save is not None:
+            modules_to_save = modules_to_save.split(',')
+        lora_rank = training_args.lora_rank
+        lora_dropout = training_args.lora_dropout
+        lora_alpha = training_args.lora_alpha
+        logger.info(f"target_modules: {target_modules}")
+        logger.info(f"lora_rank: {lora_rank}")
+        peft_config = LoraConfig(
+            task_type=TaskType.CAUSAL_LM,
+            target_modules=target_modules,
+            inference_mode=False,
+            r=lora_rank, lora_alpha=lora_alpha,
+            lora_dropout=lora_dropout,
+            modules_to_save=modules_to_save)
+        model = get_peft_model(model, peft_config)
+    model.print_trainable_parameters()
+    old_state_dict = model.state_dict
+    model.state_dict = (
+        lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict())
+    ).__get__(model, type(model))
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        data_collator=fault_tolerance_data_collator,
+        compute_metrics=compute_metrics if training_args.do_eval and not is_torch_tpu_available() else None,
+        preprocess_logits_for_metrics=preprocess_logits_for_metrics
+        if training_args.do_eval and not is_torch_tpu_available()
+        else None,
+    )
+    trainer.add_callback(SavePeftModelCallback)
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+
+        metrics = train_result.metrics
+
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+
+if __name__ == "__main__":
+    main()

04-gene-sft/.ipynb_checkpoints/run_clm_sft_with_peft-checkpoint.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+
+import logging
+import math
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+from pathlib import Path
+import datasets
+import torch
+from build_dataset import build_instruction_dataset, DataCollatorForSupervisedDataset
+import transformers
+from transformers import (
+    CONFIG_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    LlamaForCausalLM,
+    LlamaTokenizer,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    set_seed,
+)
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import send_example_telemetry
+from transformers.utils.versions import require_version
+
+from peft import LoraConfig, TaskType, get_peft_model, PeftModel, get_peft_model_state_dict
+from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+
+IGNORE_INDEX = -100
+DEFAULT_PAD_TOKEN = "[PAD]"
+DEFAULT_EOS_TOKEN = "</s>"
+DEFAULT_BOS_TOKEN = "<s>"
+DEFAULT_UNK_TOKEN = "<unk>"
+
+require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
+
+
+class SavePeftModelCallback(transformers.TrainerCallback):
+    def save_model(self, args, state, kwargs):
+        if state.best_model_checkpoint is not None:
+            checkpoint_folder = os.path.join(state.best_model_checkpoint, "sft_lora_model")
+        else:
+            checkpoint_folder = os.path.join(args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}")
+
+        peft_model_path = os.path.join(checkpoint_folder, "sft_lora_model")
+        kwargs["model"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
+
+    def on_save(self, args, state, control, **kwargs):
+        self.save_model(args, state, kwargs)
+        return control
+
+    def on_train_end(self, args, state, control, **kwargs):
+        peft_model_path = os.path.join(args.output_dir, "sft_lora_model")
+        kwargs["model"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The tokenizer for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
+                "with private models)."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    dataset_dir: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[float] = field(
+        default=0.05,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+    data_cache_dir: Optional[str] = field(default=None, metadata={"help": "The datasets processed stored"})
+
+    max_seq_length: Optional[int] = field(default=512)
+
+
+@dataclass
+class MyTrainingArguments(TrainingArguments):
+    trainable : Optional[str] = field(default="q_proj,v_proj")
+    lora_rank : Optional[int] = field(default=8)
+    lora_dropout : Optional[float] = field(default=0.1)
+    lora_alpha : Optional[float] = field(default=32.)
+    modules_to_save : Optional[str] = field(default=None)
+    peft_path : Optional[str] = field(default=None)
+    force_resize_embeddings: bool = field(default=False)
+
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, MyTrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,  # if training_args.local_rank in [-1, 0] else logging.WARN,
+        handlers=[logging.StreamHandler(sys.stdout)],)
+
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+    # transformers.tokenization_utils.logging.set_verbosity_warning()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.tokenizer_name_or_path:
+        tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+
+    if tokenizer.pad_token is None:
+        print(f"Adding pad token {DEFAULT_PAD_TOKEN}")
+        tokenizer.add_special_tokens(dict(pad_token=DEFAULT_PAD_TOKEN))
+
+    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
+    eval_dataset=None
+    train_dataset = None
+
+    if training_args.do_train:
+        with training_args.main_process_first(desc="loading and tokenization"):
+            path = Path(data_args.dataset_dir)
+            files = [os.path.join(path,file.name) for file in path.glob("*.json")]
+            logger.info(f"Training files: {' '.join(files)}")
+            train_dataset = build_instruction_dataset(
+                data_path=files,
+                tokenizer=tokenizer,
+                max_seq_length=data_args.max_seq_length,
+                data_cache_dir = None,
+                preprocessing_num_workers = data_args.preprocessing_num_workers)
+        logger.info(f"Num train_samples  {len(train_dataset)}")
+        logger.info("training example:")
+        logger.info(tokenizer.decode(train_dataset[0]['input_ids']))
+    if training_args.do_eval:
+        with training_args.main_process_first(desc="loading and tokenization"):
+            files = [data_args.validation_file]
+            logger.info(f"Evaluation files: {' '.join(files)}")
+            eval_dataset = build_instruction_dataset(
+                data_path=files,
+                tokenizer=tokenizer,
+                max_seq_length=data_args.max_seq_length,
+                data_cache_dir = None,
+                preprocessing_num_workers = data_args.preprocessing_num_workers)
+        logger.info(f"Num eval_samples  {len(eval_dataset)}")
+        logger.info("eval example:")
+        logger.info(tokenizer.decode(eval_dataset[0]['input_ids']))
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = LlamaForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            use_auth_token=True if model_args.use_auth_token else None,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=True
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    logger.info(f"len(tokenizer):{len(tokenizer)}")
+    embedding_size = model.get_input_embeddings().weight.shape[0]
+    if len(tokenizer) != embedding_size:
+        logger.info("resize the embedding size by the size of the tokenizer")
+        model.resize_token_embeddings(len(tokenizer))
+
+    if training_args.peft_path is not None:
+        logger.info("Peft from pre-trained model")
+        model = PeftModel.from_pretrained(model, training_args.peft_path)
+    else:
+        logger.info("Init new peft model")
+        target_modules = training_args.trainable.split(',')
+        modules_to_save = training_args.modules_to_save
+        if modules_to_save is not None:
+            modules_to_save = modules_to_save.split(',')
+        lora_rank = training_args.lora_rank
+        lora_dropout = training_args.lora_dropout
+        lora_alpha = training_args.lora_alpha
+        logger.info(f"target_modules: {target_modules}")
+        logger.info(f"lora_rank: {lora_rank}")
+        peft_config = LoraConfig(
+            task_type=TaskType.CAUSAL_LM,
+            target_modules=target_modules,
+            inference_mode=False,
+            r=lora_rank, lora_alpha=lora_alpha,
+            lora_dropout=lora_dropout,
+            modules_to_save=modules_to_save)
+        model = get_peft_model(model, peft_config)
+
+    #model.base_model.tie_weights()
+    model.print_trainable_parameters()
+    logger.info(f"model.modules_to_save: {model.modules_to_save}")
+    old_state_dict = model.state_dict
+    model.state_dict = (
+        lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict())
+    ).__get__(model, type(model))
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+    )
+    trainer.add_callback(SavePeftModelCallback)
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+
+        metrics = train_result.metrics
+
+        metrics["train_samples"] = len(train_dataset)
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+        metrics["eval_samples"] =len(eval_dataset)
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+
+if __name__ == "__main__":
+    main()

04-gene-sft/.ipynb_checkpoints/run_pt-checkpoint.sh

@@ -0,0 +1,55 @@
+lr=2e-4
+lora_rank=8
+lora_alpha=32
+lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
+modules_to_save="embed_tokens,lm_head"
+lora_dropout=0.05
+
+pretrained_model=./llama-7b-hf
+dna_eng_tokenizer_path=./merged_dna_eng_tokenizer_hf
+dataset_dir=./train_data
+data_cache=temp_data_cache_dir
+per_device_train_batch_size=32
+per_device_eval_batch_size=32
+gradient_accumulation_steps=8
+output_dir=dnahlm_llama_7b
+
+deepspeed_config_file=ds_zero2_no_offload.json
+
+torchrun --nnodes 1 --nproc_per_node 6 run_clm_pt_with_peft.py \
+    --deepspeed ${deepspeed_config_file} \
+    --model_name_or_path ${pretrained_model} \
+    --tokenizer_name_or_path ${dna_eng_tokenizer_path} \
+    --dataset_dir ${dataset_dir} \
+    --data_cache_dir ${data_cache} \
+    --validation_split_percentage 0.001 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --do_train \
+    --seed $RANDOM \
+    --fp16 \
+    --num_train_epochs 1 \
+    --lr_scheduler_type cosine \
+    --learning_rate ${lr} \
+    --warmup_ratio 0.05 \
+    --weight_decay 0.01 \
+    --logging_strategy steps \
+    --logging_steps 10 \
+    --save_strategy steps \
+    --save_total_limit 3 \
+    --save_steps 200 \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --preprocessing_num_workers 128 \
+    --block_size 512 \
+    --output_dir ${output_dir} \
+    --overwrite_output_dir \
+    --ddp_timeout 30000 \
+    --logging_first_step True \
+    --lora_rank ${lora_rank} \
+    --lora_alpha ${lora_alpha} \
+    --trainable ${lora_trainable} \
+    --modules_to_save ${modules_to_save} \
+    --lora_dropout ${lora_dropout} \
+    --torch_dtype float16 \
+    --gradient_checkpointing \
+    --ddp_find_unused_parameters False

04-gene-sft/.ipynb_checkpoints/run_sft-checkpoint.sh

@@ -0,0 +1,59 @@
+lr=1e-4
+lora_rank=8
+lora_alpha=32
+lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
+modules_to_save="embed_tokens,lm_head"
+lora_dropout=0.05
+
+pretrained_model=dnahlm-merge-hf
+chinese_tokenizer_path=dnahlm-merge-hf
+dataset_dir=sft_data
+per_device_train_batch_size=32
+per_device_eval_batch_size=32
+gradient_accumulation_steps=8
+output_dir=dnahlm-llama7b-sft
+#peft_model=peft_model/dir
+validation_file=val_data.json
+
+deepspeed_config_file=ds_zero2_no_offload.json
+
+torchrun --nnodes 1 --nproc_per_node 6 run_clm_sft_with_peft.py \
+    --deepspeed ${deepspeed_config_file} \
+    --model_name_or_path ${pretrained_model} \
+    --tokenizer_name_or_path ${chinese_tokenizer_path} \
+    --dataset_dir ${dataset_dir} \
+    --validation_split_percentage 0.001 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --do_train \
+    --do_eval \
+    --seed $RANDOM \
+    --fp16 \
+    --num_train_epochs 8 \
+    --lr_scheduler_type cosine \
+    --learning_rate ${lr} \
+    --warmup_ratio 0.03 \
+    --weight_decay 0 \
+    --logging_strategy steps \
+    --logging_steps 10 \
+    --save_strategy steps \
+    --save_total_limit 3 \
+    --evaluation_strategy steps \
+    --eval_steps 100 \
+    --save_steps 200 \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --preprocessing_num_workers 4 \
+    --max_seq_length 512 \
+    --output_dir ${output_dir} \
+    --overwrite_output_dir \
+    --ddp_timeout 30000 \
+    --logging_first_step True \
+    --lora_rank ${lora_rank} \
+    --lora_alpha ${lora_alpha} \
+    --trainable ${lora_trainable} \
+    --modules_to_save ${modules_to_save} \
+    --lora_dropout ${lora_dropout} \
+    --torch_dtype float16 \
+    --validation_file ${validation_file} \
+    --gradient_checkpointing \
+    --ddp_find_unused_parameters False 

04-gene-sft/1-finetue-intro.ipynb

@@ -31,6 +31,12 @@
     "\"yuanzhoulvpi/gpt2_chinese\", num_labels=2\n",
     ")\n",
     "\n",
+    "对应的训练数据一般是这样的：\n",
+    "\n",
+    "| seq                          | label |\n",
+    "|------------------------------|-------|\n",
+    "| 他家的奶茶超级好喝。。。      | 1     |\n",
+    "| 他家的奶茶超级难喝。。。      | 0     |\n",
     "\n",
     "\n",
     "2 如果是把分类问题，改成指令微调的模式，就是像\n",
@@ -174,7 +180,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "64312191-423f-4a18-aa0c-036374e93fb2",
    "metadata": {},
    "outputs": [],
@@ -192,10 +198,44 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "32c16282-f9f1-4545-b522-daf2b39b4ead",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2Model(\n",
+       "  (wte): Embedding(50257, 768)\n",
+       "  (wpe): Embedding(1024, 768)\n",
+       "  (drop): Dropout(p=0.1, inplace=False)\n",
+       "  (h): ModuleList(\n",
+       "    (0-11): 12 x GPT2Block(\n",
+       "      (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "      (attn): GPT2SdpaAttention(\n",
+       "        (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "        (c_proj): Conv1D(nf=768, nx=768)\n",
+       "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "      )\n",
+       "      (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "      (mlp): GPT2MLP(\n",
+       "        (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "        (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "        (act): NewGELUActivation()\n",
+       "        (dropout): Dropout(p=0.1, inplace=False)\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#原始模型\n",
     "from transformers import AutoModel\n",
@@ -205,10 +245,55 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "1149163f-4d89-472e-8d45-ebcbb5f9575e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "GPT2ForSequenceClassification(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (score): Linear(in_features=768, out_features=2, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#分类微调模型\n",
     "from transformers import AutoModelForSequenceClassification\n",
@@ -218,16 +303,110 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "id": "09735059-507c-48c4-893f-ca0da21ce5e8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#指令微调模型\n",
     "from transformers import AutoModelForCausalLM\n",
-    "sft_model = AutoModelForMaskedLM.from_pretrained(\"gpt2\")\n",
+    "sft_model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
     "sft_model"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d1407cbe-4996-4898-a135-e26d28da2a2a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from transformers import GPT2LMHeadModel\n",
+    "gpt2_model = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
+    "gpt2_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "92fc8e55-2d90-4694-b8df-90885d08d51a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/2-gpt2-instruction-ft.ipynb

@@ -8,6 +8,123 @@
     "# 4.2 基于GPT2的指令微调"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "02cd6e13-bbfb-413a-8236-ff092456fd1c",
+   "metadata": {},
+   "source": [
+    "我还是用第二章中的分类的例子，使用指令微调的形式，来再次解决分类问题。\n",
+    "\n",
+    "使用 GPT-2 进行文本分类的两种方法：**使用 GPT-2 的分类头（Classification Header）** 和 **将分类任务转换为指令微调**，在思路、实现、优劣势和适用场景上存在明显差异。以下是详细对比：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 核心思路**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **基本概念**                | 在 GPT-2 顶部添加一个分类头（通常是一个线性层），直接预测分类标签。 | 将分类任务转化为自然语言指令，模型通过微调理解并完成指令形式的任务。 |\n",
+    "| **实现方式**                | 修改 GPT-2 模型，添加 `num_labels` 分类头并定义分类损失函数。     | 构建任务指令数据（Instruction + Input + Output），然后微调模型。 |\n",
+    "| **数据形式**                | 文本与其分类标签的直接映射。                                   | 文本通过指令转化为生成任务。例如：<br>`Input`: 文章内容<br>`Output`: 分类结果。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 数据格式**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **数据格式**                | - 输入：文本 <br>- 标签：离散类别标签（如 0, 1, 2）。             | - 指令：自然语言描述任务（如 \"请分类以下文本\"）。<br>- 输入：分类文本。<br>- 输出：分类结果（文本形式）。 |\n",
+    "| **示例**                   | 输入：`\"This is a happy day!\"`<br>标签：`1`（表示积极）         | `Instruction`: \"请对以下文本进行情感分类\"<br>`Input`: `\"This is a happy day!\"`<br>`Output`: `\"积极\"` |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 模型结构**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **模型结构**                | - GPT-2 + 分类头（线性层）。                                   | - GPT-2 原始结构，无需额外的分类头。                   |\n",
+    "| **损失函数**                | - 使用交叉熵损失（Cross Entropy Loss）。                       | - 使用自回归的语言建模损失（Language Modeling Loss）。  |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 训练过程**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **微调对象**                | 主要微调分类头部分的参数（可选择冻结 GPT-2 的主干部分）。         | 微调整个 GPT-2 模型（或使用参数高效微调如 LoRA）。      |\n",
+    "| **标签处理**                | 离散化标签（如 0, 1, 2）。                                    | 标签转化为自然语言（如“积极”、“中立”、“消极”）。      |\n",
+    "| **训练难度**                | - 简单，标准分类任务流程。<br>- 数据需求较小，适合小规模微调。     | - 复杂，需要构造高质量的指令数据集。<br>- 数据需求较大，适合多任务场景。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 优缺点分析**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **优点**                    | - 训练速度快，计算资源需求较低。<br>- 实现简单，适合单一任务。    | - 泛化能力��，支持多任务扩展。<br>- 与多任务微调和开放式生成兼容。 |\n",
+    "| **缺点**                    | - 只能处理分类任务，难以扩展为其他任务。<br>- 需要人工调整分类头和损失函数。 | - 数据构造复杂且对数据质量依赖较高。<br>- 训练资源需求较大，训练时间较长。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 适用场景**\n",
+    "\n",
+    "| **方法**                    | **使用 GPT-2 分类头**                                          | **转换为指令微调**                                      |\n",
+    "|-----------------------------|-------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **适用场景**                | - 单任务文本分类，如情感分析、垃圾邮件检测等。                 | - 多任务场景，支持分类、翻译、摘要等任务的统一处理。     |\n",
+    "| **数据规模**                | 适合小数据集，数千到数万条数据即可训练效果良好。                  | 适合大数据集，特别是多任务、多领域的数据集。             |\n",
+    "| **需求类型**                | 专注于提高单一任务的分类准确率。                                | 需要增强模型的多任务泛化能力，同时提升用户交互体验。     |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 综合对比总结**\n",
+    "\n",
+    "| **维度**                | **使用 GPT-2 分类头**                                           | **转换为指令微调**                                      |\n",
+    "|-------------------------|--------------------------------------------------------------|-------------------------------------------------------|\n",
+    "| **实现复杂度**          | 较低，直接添加分类头并使用标准分类流程即可完成。                    | 较高，需要构造高质量指令数据，并调整训练流程。            |\n",
+    "| **资源需求**            | 较低，仅需调整分类头部分，训练时间和显存消耗较少。                   | 较高，需要微调整个模型，且对数据和算力需求更大。          |\n",
+    "| **性能表现**            | 对单一分类任务效果较好，但泛化能力较弱。                           | 在多任务、多样化分类场景中表现更强，且可扩展为其他任务类型。 |\n",
+    "| **扩展性**              | 较差，仅适用于当前任务，难以迁移到其他任务。                        | 较强，可适应多任务指令和开放式生成场景。                 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **选择建议**\n",
+    "\n",
+    "1. **使用 GPT-2 分类头**：\n",
+    "   - 如果任务是单一分类问题（如情感分析、垃圾邮件检测），并且数据量有限，推荐使用分类头方法。\n",
+    "   - 适合快速实现和部署，无需复杂的预处理和指令数据集构建。\n",
+    "\n",
+    "2. **转换为指令微调**：\n",
+    "   - 如果任务需要多样化（分类+生成+翻译等），或需要对未见任务有更好的泛化能力，推荐使用指令微调。\n",
+    "   - 适合多任务、多场景部署，尤其是在 ChatGPT 风格的应用中更为适用。\n",
+    "\n",
+    "通过综合任务需求、数据规模和资源条件选择合适的方法，能够有效提升模型性能并实现更广泛的适用性。\n",
+    "\n",
+    "\n",
+    "原始的数据格式如下：\n",
+    "| sequence                                               | label | label_name     |\n",
+    "|--------------------------------------------------------|-------|----------------|\n",
+    "| TATATTTTCTCAGCTGAGTTAATTAGTTTCACTAGTTAACTGAGAATAAAAGAA | 1     | promoter       |\n",
+    "| TGGGGAGGGTCCGGTGTTAGTTAGATACATCCCCAGACCCACACCCCGGATAGA | 0     | Non-promoter   |\n",
+    "\n",
+    "转成指令的格式为：\n",
+    "```\n",
+    "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.', \n",
+    "'input': 'CATGCGGGTCG...', \n",
+    "'output': 'Non-promoter'}\n",
+    "```\n",
+    "\n",
+    "然后写成指令微调数据格式，当做一般的文本进行训练：\n",
+    "```\n",
+    "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+    "### Instruction:\n",
+    "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+    "### Input:\n",
+    "TCTTTCTCTTCTGTATCATTCTACTT...\n",
+    "### Response:\n",
+    "Non-promoter\n",
+    "```\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

04-gene-sft/3-llama-expand-dict.ipynb

@@ -114,10 +114,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "19a06b82-31b8-48cb-9c83-ec016da2da8a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['▁TCG', 'ACGGC', 'ACGCG', 'ACAGC', 'AGCG', 'AGCCCC', 'GCGC', 'ACCCG', 'AGCGCG', 'AKCG', 'FVGP', 'MV', 'HLKV', 'HLE', 'ADV', 'ASSC', 'RS', 'AVI', 'YL', 'TS', 'EEP', 'FEG', 'VLGL', 'RLKE', 'GI', 'AI', 'TGC', 'WPR', 'WP', 'DEM', 'DE', 'RS', 'AVW', 'RV', 'EPY', 'TR', 'HFG', 'RVL', 'YS', 'FGV']\n"
+     ]
+    }
+   ],
    "source": [
     "from sentencepiece import SentencePieceProcessor\n",
     "model_path = \"gene_bpe_seg.model\"\n",
@@ -147,7 +155,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "3bafcc33-2923-4026-bc39-c6ec716d2e3c",
    "metadata": {},
    "outputs": [],
@@ -161,10 +169,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "66cb86ed-3225-4bb0-8aca-6005bc918d03",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "32000 60000\n",
+      "['<s>', '</s>', '<unk>']\n",
+      "[1, 2, 0]\n",
+      "{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}\n"
+     ]
+    }
+   ],
    "source": [
     "llama_tokenizer_dir = \"llama-7b-hf\" \n",
     "dna_sp_model_file = \"gene_bpe_seg.model\"\n",
@@ -188,10 +214,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "7ba4240e-bc08-4be0-8ca3-c4e7a47fa055",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "32000\n",
+      "Before:32000\n",
+      "New model pieces: 91643\n"
+     ]
+    }
+   ],
    "source": [
     "## Add dna tokens to LLaMA tokenizer\n",
     "llama_spm_tokens_set=set(p.piece for p in llama_spm.pieces)\n",
@@ -210,10 +246,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "a240a7d8-c1a9-4473-a5c5-157a25f97c16",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gene-LLaMA tokenizer has been saved to merged_gene_eng_tokenizer_hf\n"
+     ]
+    }
+   ],
    "source": [
     "## Save\n",
     "output_sp_dir = 'merged_gene_eng_tokenizer_sp'\n",
@@ -229,10 +273,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "cbd1f648-f8a0-4f16-b516-2ce3e7c7cfee",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['<s>', '</s>', '<unk>']\n",
+      "[1, 2, 0]\n",
+      "{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}\n",
+      "Test text:\n",
+      " TCGACGGCACGCGACAGCAGCGAGCCCCGCGCACCCGAGCGCGAKCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+      "The primary use of LLaMA is research on large language models, including\n",
+      "Tokenized by LLaMA tokenizer:['▁T', 'CG', 'AC', 'G', 'GC', 'AC', 'GC', 'G', 'AC', 'AG', 'CA', 'GC', 'G', 'AG', 'CC', 'CC', 'GC', 'GC', 'AC', 'CC', 'GA', 'GC', 'GC', 'GA', 'K', 'CG', 'F', 'V', 'G', 'PM', 'V', 'HL', 'K', 'V', 'H', 'LE', 'AD', 'VA', 'SS', 'CR', 'S', 'AV', 'I', 'Y', 'LT', 'SEE', 'PF', 'EG', 'V', 'L', 'GL', 'RL', 'KE', 'G', 'IA', 'IT', 'GC', 'W', 'PR', 'WP', 'DE', 'MD', 'ERS', 'AV', 'WR', 'VE', 'PY', 'TR', 'H', 'F', 'GR', 'V', 'LY', 'SF', 'GV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n",
+      "Tokenized by GENE-LLaMA tokenizer:['▁TCG', 'ACGGC', 'ACGCG', 'ACAG', 'CA', 'GCG', 'AGCCCC', 'GCGC', 'ACCCG', 'AGCGCG', 'AKCG', 'FVGP', 'MVHL', 'KV', 'HLE', 'ADV', 'ASSC', 'RSAV', 'I', 'YL', 'TSEE', 'P', 'FEG', 'VLGL', 'RLK', 'EGI', 'AI', 'TGC', 'W', 'PRW', 'P', 'DEM', 'DER', 'SAV', 'W', 'RVE', 'PY', 'TRH', 'FG', 'RVLY', 'SFGV', ',', '<0x0A>', 'The', '▁primary', '▁use', '▁of', '▁L', 'La', 'MA', '▁is', '▁research', '▁on', '▁large', '▁language', '▁models', ',', '▁including']\n"
+     ]
+    }
+   ],
    "source": [
     "# Test\n",
     "llama_tokenizer = LlamaTokenizer.from_pretrained(llama_tokenizer_dir)\n",
@@ -246,6 +305,14 @@
     "print(f\"Tokenized by LLaMA tokenizer:{llama_tokenizer.tokenize(text)}\")\n",
     "print(f\"Tokenized by GENE-LLaMA tokenizer:{dna_llama_tokenizer.tokenize(text)}\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46ae7605-2ef8-4927-bff3-2c0325f8df0d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/4-deepspeed-intro.ipynb

@@ -56,6 +56,8 @@
     "\n",
     "每个阶段都进一步减少显存需求，Stage 3 可支持超大规模模型（如 GPT-3）。\n",
     "\n",
+    "<img src='img/deepspeed.png' width='600px' />\n",
+    "\n",
     "#### **（2）混合精度训练**\n",
     "通过 FP16 或 BF16（半精度浮点数）计算，显著减少显存占用并提升计算效率。\n",
     "\n",
@@ -567,6 +569,14 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce701aeb-c8c7-450a-bbf9-b793a19cd0c6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-gene-sft/5-peft-intro.ipynb

@@ -0,0 +1,870 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "963e9ae0-ac68-44be-8c7d-fb9842784362",
+   "metadata": {},
+   "source": [
+    "# 4.5 peft简介"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "182b82c4-d484-4c15-a600-03c3b51367ec",
+   "metadata": {},
+   "source": [
+    "**PEFT**（Parameter-Efficient Fine-Tuning，参数高效微调）是一种优化技术，旨在以最小的参数更新实现对大规模预训练模型（如 GPT、BERT 等）的微调。PEFT 技术通过减少微调所需的参数量，显著降低了存储和计算开销，同时保留模型的性能，特别适合资源受限的场景和领域特定任务的定制化。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 核心思想**\n",
+    "传统的微调方式需要更新整个预训练模型的所有参数，PEFT 技术通过只调整少量的参数（如特定层或额外添加的小型模块）实现微调目标，大幅减少了训练开销和存储需求。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 常见的 PEFT 方法**\n",
+    "\n",
+    "#### **（1）Adapter 模型**\n",
+    "- 在每一层 Transformer 的输出中插入小型适配器模块，仅训练适配器模块的参数。\n",
+    "- 原始模型参数保持冻结不变。\n",
+    "- 优点：适配器模块参数量小，能适应不同任务。\n",
+    "\n",
+    "示例方法：\n",
+    "- **AdapterFusion**\n",
+    "- **MAD-X**\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（2）Prefix Tuning**\n",
+    "- 在 Transformer 的输入前添加一组可学习的前缀向量，这些前缀与模型的注意力机制交互。\n",
+    "- 只调整前缀向量的参数，而不更新原始模型。\n",
+    "- 优点：对生成任务效果显著，参数量进一步减少。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（3）LoRA（Low-Rank Adaptation）**\n",
+    "- 将预训练模型中的部分权重分解为两个低秩矩阵，仅调整这些低秩矩阵的参数。\n",
+    "- 原始权重保持冻结状态。\n",
+    "- 优点：参数量极小，计算高效。\n",
+    "  \n",
+    "---\n",
+    "\n",
+    "#### **（4）Prompt Tuning**\n",
+    "- 在输入文本中添加可学习的提示（Prompt）。\n",
+    "- 适合 NLP 任务中的文本生成、分类等。\n",
+    "- 优点：实现简单，易于集成到现有框架。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. PEFT 的优势**\n",
+    "\n",
+    "1. **显著减少参数更新量**：\n",
+    "   - 微调传统的大模型（如 GPT-3）需要更新数百亿参数，而 PEFT 仅需更新百万级别甚至更少的参数。\n",
+    "\n",
+    "2. **高效存储**：\n",
+    "   - 每个任务的微调结果只需存储少量额外参数，而不是整个模型。\n",
+    "\n",
+    "3. **适用多任务**：\n",
+    "   - 同一预训练模型可以通过不同的 PEFT 模块适配多个任务，无需重新训练。\n",
+    "\n",
+    "4. **降低计算开销**：\n",
+    "   - 训练所需的内存和计算显著减少，适合资源有限的环境。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 应用场景**\n",
+    "\n",
+    "1. **领域特定任务**：\n",
+    "   - 医疗、法律、金融等领域微调预训练模型。\n",
+    "\n",
+    "2. **多任务学习**：\n",
+    "   - 适配多个任务，复用同一模型的预训练权重。\n",
+    "\n",
+    "3. **资源受限场景**：\n",
+    "   - 移动设备、边缘设备上的模型部署。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. Hugging Face PEFT 库**\n",
+    "\n",
+    "Hugging Face 提供了专门的 PEFT 库，支持多种参数高效微调技术：\n",
+    "- **安装**：\n",
+    "  ```bash\n",
+    "  pip install peft\n",
+    "  ```\n",
+    "- **使用 LoRA 微调示例**：\n",
+    "  ```python\n",
+    "  from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "  from peft import LoraConfig, get_peft_model, TaskType\n",
+    "\n",
+    "  # 加载模型和分词器\n",
+    "  model_name = \"gpt2\"\n",
+    "  model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "  tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "\n",
+    "  # 配置 LoRA\n",
+    "  lora_config = LoraConfig(\n",
+    "      task_type=TaskType.CAUSAL_LM,\n",
+    "      r=8,\n",
+    "      lora_alpha=32,\n",
+    "      target_modules=[\"q_proj\", \"v_proj\"],\n",
+    "      lora_dropout=0.1,\n",
+    "      bias=\"none\"\n",
+    "  )\n",
+    "\n",
+    "  # 使用 LoRA 微调模型\n",
+    "  model = get_peft_model(model, lora_config)\n",
+    "  model.print_trainable_parameters()\n",
+    "\n",
+    "  # 微调代码...\n",
+    "  ```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. PEFT 的局限性**\n",
+    "1. **特定任务限制**：\n",
+    "   - 在一些复杂任务中，PEFT 方法可能不如全量微调效果好。\n",
+    "\n",
+    "2. **需要设计合适的模块**：\n",
+    "   - 不同任务需要选择和设计合适的 PEFT 技术。\n",
+    "\n",
+    "3. **与模型架构相关**：\n",
+    "   - PEFT 技术可能需要对模型架构进行一定程度的修改。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 小结**\n",
+    "PEFT 是一个极具潜力的技术，特别适合在有限资源下对大模型进行微调。它在许多领域和任务中已显示出良好的效果，例如 LoRA 和 Adapter 模型已经成为高效微调的主流方法。\n",
+    "\n",
+    "如果您需要实现高效微调，可以结合 Hugging Face 的 PEFT 库快速上手。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5aa3d240-44e1-4811-8f61-d6ff2500a798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
+   "metadata": {},
+   "source": [
+    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "41a0c049-9134-4d89-aad0-1aa2241a9fca",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4becc479adbc472bb7672d49da16aafd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "transformer\n",
+      "transformer.wte\n",
+      "transformer.wpe\n",
+      "transformer.drop\n",
+      "transformer.h\n",
+      "transformer.h.0\n",
+      "transformer.h.0.ln_1\n",
+      "transformer.h.0.attn\n",
+      "transformer.h.0.attn.c_attn\n",
+      "transformer.h.0.attn.c_proj\n",
+      "transformer.h.0.attn.attn_dropout\n",
+      "transformer.h.0.attn.resid_dropout\n",
+      "transformer.h.0.ln_2\n",
+      "transformer.h.0.mlp\n",
+      "transformer.h.0.mlp.c_fc\n",
+      "transformer.h.0.mlp.c_proj\n",
+      "transformer.h.0.mlp.act\n",
+      "transformer.h.0.mlp.dropout\n",
+      "transformer.h.1\n",
+      "transformer.h.1.ln_1\n",
+      "transformer.h.1.attn\n",
+      "transformer.h.1.attn.c_attn\n",
+      "transformer.h.1.attn.c_proj\n",
+      "transformer.h.1.attn.attn_dropout\n",
+      "transformer.h.1.attn.resid_dropout\n",
+      "transformer.h.1.ln_2\n",
+      "transformer.h.1.mlp\n",
+      "transformer.h.1.mlp.c_fc\n",
+      "transformer.h.1.mlp.c_proj\n",
+      "transformer.h.1.mlp.act\n",
+      "transformer.h.1.mlp.dropout\n",
+      "transformer.h.2\n",
+      "transformer.h.2.ln_1\n",
+      "transformer.h.2.attn\n",
+      "transformer.h.2.attn.c_attn\n",
+      "transformer.h.2.attn.c_proj\n",
+      "transformer.h.2.attn.attn_dropout\n",
+      "transformer.h.2.attn.resid_dropout\n",
+      "transformer.h.2.ln_2\n",
+      "transformer.h.2.mlp\n",
+      "transformer.h.2.mlp.c_fc\n",
+      "transformer.h.2.mlp.c_proj\n",
+      "transformer.h.2.mlp.act\n",
+      "transformer.h.2.mlp.dropout\n",
+      "transformer.h.3\n",
+      "transformer.h.3.ln_1\n",
+      "transformer.h.3.attn\n",
+      "transformer.h.3.attn.c_attn\n",
+      "transformer.h.3.attn.c_proj\n",
+      "transformer.h.3.attn.attn_dropout\n",
+      "transformer.h.3.attn.resid_dropout\n",
+      "transformer.h.3.ln_2\n",
+      "transformer.h.3.mlp\n",
+      "transformer.h.3.mlp.c_fc\n",
+      "transformer.h.3.mlp.c_proj\n",
+      "transformer.h.3.mlp.act\n",
+      "transformer.h.3.mlp.dropout\n",
+      "transformer.h.4\n",
+      "transformer.h.4.ln_1\n",
+      "transformer.h.4.attn\n",
+      "transformer.h.4.attn.c_attn\n",
+      "transformer.h.4.attn.c_proj\n",
+      "transformer.h.4.attn.attn_dropout\n",
+      "transformer.h.4.attn.resid_dropout\n",
+      "transformer.h.4.ln_2\n",
+      "transformer.h.4.mlp\n",
+      "transformer.h.4.mlp.c_fc\n",
+      "transformer.h.4.mlp.c_proj\n",
+      "transformer.h.4.mlp.act\n",
+      "transformer.h.4.mlp.dropout\n",
+      "transformer.h.5\n",
+      "transformer.h.5.ln_1\n",
+      "transformer.h.5.attn\n",
+      "transformer.h.5.attn.c_attn\n",
+      "transformer.h.5.attn.c_proj\n",
+      "transformer.h.5.attn.attn_dropout\n",
+      "transformer.h.5.attn.resid_dropout\n",
+      "transformer.h.5.ln_2\n",
+      "transformer.h.5.mlp\n",
+      "transformer.h.5.mlp.c_fc\n",
+      "transformer.h.5.mlp.c_proj\n",
+      "transformer.h.5.mlp.act\n",
+      "transformer.h.5.mlp.dropout\n",
+      "transformer.h.6\n",
+      "transformer.h.6.ln_1\n",
+      "transformer.h.6.attn\n",
+      "transformer.h.6.attn.c_attn\n",
+      "transformer.h.6.attn.c_proj\n",
+      "transformer.h.6.attn.attn_dropout\n",
+      "transformer.h.6.attn.resid_dropout\n",
+      "transformer.h.6.ln_2\n",
+      "transformer.h.6.mlp\n",
+      "transformer.h.6.mlp.c_fc\n",
+      "transformer.h.6.mlp.c_proj\n",
+      "transformer.h.6.mlp.act\n",
+      "transformer.h.6.mlp.dropout\n",
+      "transformer.h.7\n",
+      "transformer.h.7.ln_1\n",
+      "transformer.h.7.attn\n",
+      "transformer.h.7.attn.c_attn\n",
+      "transformer.h.7.attn.c_proj\n",
+      "transformer.h.7.attn.attn_dropout\n",
+      "transformer.h.7.attn.resid_dropout\n",
+      "transformer.h.7.ln_2\n",
+      "transformer.h.7.mlp\n",
+      "transformer.h.7.mlp.c_fc\n",
+      "transformer.h.7.mlp.c_proj\n",
+      "transformer.h.7.mlp.act\n",
+      "transformer.h.7.mlp.dropout\n",
+      "transformer.h.8\n",
+      "transformer.h.8.ln_1\n",
+      "transformer.h.8.attn\n",
+      "transformer.h.8.attn.c_attn\n",
+      "transformer.h.8.attn.c_proj\n",
+      "transformer.h.8.attn.attn_dropout\n",
+      "transformer.h.8.attn.resid_dropout\n",
+      "transformer.h.8.ln_2\n",
+      "transformer.h.8.mlp\n",
+      "transformer.h.8.mlp.c_fc\n",
+      "transformer.h.8.mlp.c_proj\n",
+      "transformer.h.8.mlp.act\n",
+      "transformer.h.8.mlp.dropout\n",
+      "transformer.h.9\n",
+      "transformer.h.9.ln_1\n",
+      "transformer.h.9.attn\n",
+      "transformer.h.9.attn.c_attn\n",
+      "transformer.h.9.attn.c_proj\n",
+      "transformer.h.9.attn.attn_dropout\n",
+      "transformer.h.9.attn.resid_dropout\n",
+      "transformer.h.9.ln_2\n",
+      "transformer.h.9.mlp\n",
+      "transformer.h.9.mlp.c_fc\n",
+      "transformer.h.9.mlp.c_proj\n",
+      "transformer.h.9.mlp.act\n",
+      "transformer.h.9.mlp.dropout\n",
+      "transformer.h.10\n",
+      "transformer.h.10.ln_1\n",
+      "transformer.h.10.attn\n",
+      "transformer.h.10.attn.c_attn\n",
+      "transformer.h.10.attn.c_proj\n",
+      "transformer.h.10.attn.attn_dropout\n",
+      "transformer.h.10.attn.resid_dropout\n",
+      "transformer.h.10.ln_2\n",
+      "transformer.h.10.mlp\n",
+      "transformer.h.10.mlp.c_fc\n",
+      "transformer.h.10.mlp.c_proj\n",
+      "transformer.h.10.mlp.act\n",
+      "transformer.h.10.mlp.dropout\n",
+      "transformer.h.11\n",
+      "transformer.h.11.ln_1\n",
+      "transformer.h.11.attn\n",
+      "transformer.h.11.attn.c_attn\n",
+      "transformer.h.11.attn.c_proj\n",
+      "transformer.h.11.attn.attn_dropout\n",
+      "transformer.h.11.attn.resid_dropout\n",
+      "transformer.h.11.ln_2\n",
+      "transformer.h.11.mlp\n",
+      "transformer.h.11.mlp.c_fc\n",
+      "transformer.h.11.mlp.c_proj\n",
+      "transformer.h.11.mlp.act\n",
+      "transformer.h.11.mlp.dropout\n",
+      "transformer.ln_f\n",
+      "lm_head\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForCausalLM\n",
+    "\n",
+    "model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
+    "\n",
+    "# 打印所有模块名称\n",
+    "for name, module in model.named_modules():\n",
+    "    print(name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37aa6abb-ab1c-4e9c-b968-579dd74044db",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0add2f79-f35c-4638-80bb-0d8a87a9b6a7",
+   "metadata": {},
+   "source": [
+    "在选择 `target_modules` 时，通常会根据模块的名称选择模型的特定部分，通常使用列表中最后一个点 `.` 后的字段名或整个路径名（如果需要更精确）。以下是对这些模块的详细分析和选择建议：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 分析模块结构**\n",
+    "\n",
+    "从列表中可以看出，GPT-2 的模块层次分为以下几类：\n",
+    "\n",
+    "1. **Embedding 层**：\n",
+    "   - `transformer.wte`：词嵌入层（Word Token Embeddings）。\n",
+    "   - `transformer.wpe`：位置嵌入层（Position Embeddings）。\n",
+    "\n",
+    "2. **Transformer 编码器层**：\n",
+    "   - 每层编号为 `transformer.h.<层号>`（共 12 层）。\n",
+    "   - 每层中包含：\n",
+    "     - **层归一化**：\n",
+    "       - `transformer.h.<层号>.ln_1`：第一层归一化。\n",
+    "       - `transformer.h.<层号>.ln_2`：第二层归一化。\n",
+    "     - **自注意力模块**：\n",
+    "       - `transformer.h.<层号>.attn.c_attn`：注意力模块的 Query、Key 和 Value 投影。\n",
+    "       - `transformer.h.<层号>.attn.c_proj`：注意力的输出投影。\n",
+    "       - `transformer.h.<层号>.attn.attn_dropout`：注意力的 Dropout。\n",
+    "       - `transformer.h.<层号>.attn.resid_dropout`：残差的 Dropout。\n",
+    "     - **前馈网络模块（MLP）**：\n",
+    "       - `transformer.h.<层号>.mlp.c_fc`：MLP 的第一层全连接。\n",
+    "       - `transformer.h.<层号>.mlp.c_proj`：MLP 的第二层全连接（输出投影）。\n",
+    "       - `transformer.h.<层号>.mlp.act`：激活函数（如 GELU）。\n",
+    "       - `transformer.h.<层号>.mlp.dropout`：MLP 的 Dropout。\n",
+    "\n",
+    "3. **最终层**：\n",
+    "   - `transformer.ln_f`：最终层归一化（LayerNorm）。\n",
+    "   - `lm_head`：语言建模头，用于生成预测的 token 分布。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 如何选择 `target_modules`**\n",
+    "\n",
+    "#### **（1）常见目标模块**\n",
+    "- `transformer.h.<层号>.attn.c_attn`：对自注意力模块的 Query、Key 和 Value 投影层微调。\n",
+    "- `transformer.h.<层号>.attn.c_proj`：对注意力输出的投影层微调。\n",
+    "- `transformer.h.<层号>.mlp.c_fc`：对前馈网络的输入全连接层微调。\n",
+    "- `transformer.h.<层号>.mlp.c_proj`：对前馈网络的输出投影层微调。\n",
+    "\n",
+    "#### **（2）推荐设置**\n",
+    "- **文本生成任务**：\n",
+    "  ```python\n",
+    "  target_modules = [\"transformer.h.*.attn.c_attn\", \"transformer.h.*.attn.c_proj\"]\n",
+    "  ```\n",
+    "  解释：\n",
+    "  - `*.attn.c_attn`：调整 Query、Key、Value 的生成。\n",
+    "  - `*.attn.c_proj`：调整注意力输出。\n",
+    "\n",
+    "- **文本分类任务**：\n",
+    "  ```python\n",
+    "  target_modules = [\"transformer.h.*.attn.c_attn\"]\n",
+    "  ```\n",
+    "  解释：\n",
+    "  - 微调自注意力模块最重要的部分即可。\n",
+    "\n",
+    "- **特定任务需要更细粒度控制**：\n",
+    "  - 仅微调某几层：\n",
+    "    ```python\n",
+    "    target_modules = [\"transformer.h.0.attn.c_attn\", \"transformer.h.0.mlp.c_fc\"]\n",
+    "    ```\n",
+    "\n",
+    "#### **（3）通配符选择**\n",
+    "使用 `*` 通配符可以指定所有层的某些模块：\n",
+    "- `transformer.h.*.attn.c_attn`：所有层的 Query、Key 和 Value 投影。\n",
+    "- `transformer.h.*.mlp.*`：所有层的 MLP 模块。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 示例：指定多个模块**\n",
+    "\n",
+    "```python\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.CAUSAL_LM,\n",
+    "    r=8,\n",
+    "    lora_alpha=32,\n",
+    "    target_modules=[\n",
+    "        \"transformer.h.*.attn.c_attn\",\n",
+    "        \"transformer.h.*.mlp.c_fc\"\n",
+    "    ],\n",
+    "    lora_dropout=0.1,\n",
+    "    bias=\"none\"\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "- 这表示对所有层的 `attn.c_attn` 和 `mlp.c_fc` 模块进行 LoRA 微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 小提示：如何确定适合的模块**\n",
+    "\n",
+    "1. **任务相关性**：\n",
+    "   - 文本生成：优先选择自注意力模块（如 `c_attn`）。\n",
+    "   - 文本分类：通常需要全局语义表示，选择 `attn.c_attn` 或 `mlp.c_fc`。\n",
+    "\n",
+    "2. **性能与资源平衡**：\n",
+    "   - 如果显存有限，可以只微调部分层。例如，仅选择浅层和深层的模块：\n",
+    "     ```python\n",
+    "     target_modules = [\"transformer.h.0.attn.c_attn\", \"transformer.h.11.attn.c_attn\"]\n",
+    "     ```\n",
+    "\n",
+    "3. **打印模块名称以调试**：\n",
+    "   - 确保选择的 `target_modules` 在模型中实际存在：\n",
+    "     ```python\n",
+    "     for name, _ in model.named_modules():\n",
+    "         if \"c_attn\" in name:\n",
+    "             print(name)\n",
+    "     ```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **建议**\n",
+    "- 一般情况下，`c_attn` 和 `c_proj` 是首选模块。\n",
+    "- 使用 `transformer.h.*` 通配符可以轻松指定多层。\n",
+    "- 根据任务需求和资源限制灵活调整目标模块，以实现最佳性能和效率。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b4a41750-420f-49c4-845d-69db394794f9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10c99eb9-8007-4297-972e-7be71768c9c3",
+   "metadata": {},
+   "source": [
+    "以下是对 `LoraConfig` 配置的更详细解释，特别是如何设置微调哪些参数、冻结哪些参数，以及一般如何选择这些设置：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. `LoraConfig` 参数解析**\n",
+    "\n",
+    "```python\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.SEQ_CLS,  # 序列分类任务\n",
+    "    r=8,                         # 降低矩阵秩\n",
+    "    lora_alpha=32,               # LoRA 的 alpha 超参数\n",
+    "    target_modules=[\"c_attn\"],   # GPT-2 中的自注意力模块\n",
+    "    lora_dropout=0.1,            # dropout 概率\n",
+    "    bias=\"none\",                 # 是否微调偏置参数\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "#### **（1）`task_type`**\n",
+    "- 定义任务类型，用于指导 PEFT 的具体行为。\n",
+    "- **常见选项**：\n",
+    "  - `TaskType.CAUSAL_LM`：自回归语言建模（��� GPT 系列模型）。\n",
+    "  - `TaskType.SEQ_CLS`：序列分类（如情感分析）。\n",
+    "  - `TaskType.TOKEN_CLS`：标注任务（如命名实体识别）。\n",
+    "  - `TaskType.SEQ_2_SEQ_LM`：序列到序列任务（如翻译、摘要）。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `TaskType.SEQ_CLS` 表示目标是文本分类任务。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（2）`r`**\n",
+    "- 表示 LoRA 的 **秩**（rank），是降低矩阵秩的核心参数。\n",
+    "- LoRA 通过将模型的权重分解为两个低秩矩阵（`A` 和 `B`），只更新这两个矩阵。\n",
+    "- `r` 的值越大，微调能力越强，但需要的额外参数也越多。\n",
+    "- **典型范围**：`4` 至 `64`，大多数任务中 `8` 或 `16` 是常用值。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `r=8` 表示使用低秩分解，并微调 8 维的参数矩阵。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（3）`lora_alpha`**\n",
+    "- 是 LoRA 的一个缩放因子，用于调节两个低秩矩阵的更新速率。\n",
+    "- **公式**：实际更新 = LoRA 输出 × `lora_alpha / r`\n",
+    "- **典型范围**：`16` 至 `128`，较大任务中可以选择更高的值。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `lora_alpha=32`，表示适中幅度的更新速率。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（4）`target_modules`**\n",
+    "- 指定要应用 LoRA 微调的模块。\n",
+    "- **常见选择**：\n",
+    "  - 对 Transformer 模型中的 **注意力模块**（如 `query`、`key`、`value`）进行微调，因为这些模块对任务性能影响较大。\n",
+    "  - 对 GPT-2，通常选择 `c_attn`（GPT-2 中负责自注意力机制的组合模块）。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `target_modules=[\"c_attn\"]` 表示只对 GPT-2 的自注意力模块 `c_attn` 应用 LoRA。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（5）`lora_dropout`**\n",
+    "- 表示 LoRA 层的 dropout 概率，用于防止过拟合。\n",
+    "- **典型范围**：`0.0` 至 `0.1`，视任务复杂性而定。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `lora_dropout=0.1`，表示有 10% 的概率随机丢弃 LoRA 层的输出。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "#### **（6）`bias`**\n",
+    "- 决定是否微调偏置参数。\n",
+    "- **选项**：\n",
+    "  - `\"none\"`：不微调任何偏置。\n",
+    "  - `\"all\"`：微调所有偏置。\n",
+    "  - `\"lora_only\"`：只微调 LoRA 层的偏置。\n",
+    "\n",
+    "**当前设置**：\n",
+    "- `bias=\"none\"`，表示所有偏置参数保持冻结。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 微调哪些参数，冻结哪些参数**\n",
+    "\n",
+    "LoRA 的核心思想是通过 **分解矩阵**，只更新少量参数，而冻结模型的大部分参数。以下是常见设置的说明：\n",
+    "\n",
+    "#### **微调的参数**\n",
+    "- LoRA 通过 `target_modules` 指定的模块，例如：\n",
+    "  - GPT-2 的 `c_attn`（自注意力模块）。\n",
+    "  - BERT 的 `query` 和 `key`。\n",
+    "- 这些模块是模型中对性能贡献最大的部分，通过微调这些模块，任务性能可以显著提升。\n",
+    "\n",
+    "#### **冻结的参数**\n",
+    "- 除了 `target_modules` 中指定的参数外，所有其他模型参数默认冻结，包括：\n",
+    "  - 预训练权重的绝大部分。\n",
+    "  - 偏置参数（如果 `bias=\"none\"`）。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 一般如何设置**\n",
+    "\n",
+    "#### **（1）针对不同任务调整**\n",
+    "- **文本分类任务**：\n",
+    "  - 优先选择自注意力模块（如 `c_attn`）作为 `target_modules`。\n",
+    "  - `r=8` 或 `r=16` 是常见选择，适中计算开销。\n",
+    "  - 设置适当的 dropout（如 `lora_dropout=0.1`）以防止过拟合。\n",
+    "  \n",
+    "- **语言生成任务**：\n",
+    "  - 对 GPT-2 或 GPT-3，选择 `q_proj` 和 `v_proj`（query 和 value 投影模块）。\n",
+    "  - `r=16` 或更高，适应生成任务的高复杂性。\n",
+    "\n",
+    "- **命名实体识别任务**：\n",
+    "  - 优先选择 `q_proj` 和 `k_proj`（query 和 key 模块）。\n",
+    "\n",
+    "#### **（2）参数量与显存的权衡**\n",
+    "- 如果显存有限，减少 `r` 的值。\n",
+    "- 对小型任务，`r=4` 或 `r=8` 通常已经足够。\n",
+    "\n",
+    "#### **（3）偏置设置**\n",
+    "- 偏置参数的影响较小，在大多数情况下，可以选择 `bias=\"none\"` 保持冻结。\n",
+    "- 对非常依赖偏置的任务（如生成风格微调），可以尝试 `bias=\"lora_only\"`。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 示例：如何选择目标模块**\n",
+    "\n",
+    "#### **GPT-2**\n",
+    "对 GPT-2 来说，以下模块通常是微调的目标：\n",
+    "- **`c_attn`**：注意力模块的组合层。\n",
+    "- **`q_proj` 和 `v_proj`**：Query 和 Value 的线性投影。\n",
+    "\n",
+    "#### **BERT**\n",
+    "对 BERT 来说，以下模块通常是微调的目标：\n",
+    "- **`query`**：Attention 的 Query 模块。\n",
+    "- **`key`**：Attention 的 Key 模块。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 总结建议**\n",
+    "- **微调的参数**：优先选择模型中注意力相关模块。\n",
+    "- **冻结的参数**：大部分参数默认冻结以节省显存。\n",
+    "- **配置选择**：根据任务复杂性调整 `r` 和 `target_modules`。\n",
+    "- **推荐起点**：\n",
+    "  - 文本分类：`target_modules=[\"c_attn\"]`, `r=8`, `lora_dropout=0.1`。\n",
+    "  - 文本生成：`target_modules=[\"q_proj\", \"v_proj\"]`, `r=16`, `lora_dropout=0.1`。\n",
+    "\n",
+    "通过这些设置，LoRA 可以在参数量极小的情况下实现高效微调，适合各种任务场景。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26d9f362-18cc-471f-b208-f29a6933c06a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer\n",
+    "from peft import LoraConfig, get_peft_model, TaskType\n",
+    "from datasets import load_dataset\n",
+    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
+    "\n",
+    "# **1. 加载模型和分词器**\n",
+    "model_name = \"gpt2\"  # 基础模型\n",
+    "num_labels = 2       # 二分类任务\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "tokenizer.pad_token = tokenizer.eos_token  # 设置 pad_token 为 eos_token\n",
+    "\n",
+    "# **2. 定义数据集**\n",
+    "# 示例数据集：dna_promoter_300\n",
+    "dataset = load_dataset(\"dnagpt/dna_promoter_300\")['train'].train_test_split(test_size=0.1)\n",
+    "\n",
+    "# **3. 数据预处理**\n",
+    "def preprocess_function(examples):\n",
+    "    examples['label'] = [int(item) for item in examples['label']]\n",
+    "    return tokenizer(\n",
+    "        examples[\"sequence\"], truncation=True, padding=\"max_length\", max_length=128\n",
+    "    )\n",
+    "\n",
+    "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
+    "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")  # Hugging Face Trainer 要求标签列名为 'labels'\n",
+    "\n",
+    "# **4. 划分数据集**\n",
+    "train_dataset = tokenized_datasets[\"train\"]\n",
+    "test_dataset = tokenized_datasets[\"test\"]\n",
+    "\n",
+    "# **5. 配置 LoRA**\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.SEQ_CLS,  # 序列分类任务\n",
+    "    r=8,                         # 降低矩阵秩\n",
+    "    lora_alpha=32,               # LoRA 的 alpha 超参数\n",
+    "    target_modules=[\"c_attn\"],   # GPT-2 中的自注意力模块\n",
+    "    lora_dropout=0.1,            # dropout 概率\n",
+    "    bias=\"none\",                 # 是否微调偏置参数\n",
+    ")\n",
+    "\n",
+    "# 使用 LoRA 包装模型\n",
+    "model = get_peft_model(model, lora_config)\n",
+    "model.print_trainable_parameters()  # 打印可训练的参数信息\n",
+    "\n",
+    "# **6. 计算指标**\n",
+    "def compute_metrics(eval_pred):\n",
+    "    predictions, labels = eval_pred\n",
+    "    preds = predictions.argmax(axis=-1)\n",
+    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=\"binary\")\n",
+    "    acc = accuracy_score(labels, preds)\n",
+    "    return {\"accuracy\": acc, \"precision\": precision, \"recall\": recall, \"f1\": f1}\n",
+    "\n",
+    "# **7. 定义训练参数**\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2_lora_text_classification\",  # 模型保存路径\n",
+    "    evaluation_strategy=\"epoch\",                 # 每个 epoch 评估一次\n",
+    "    save_strategy=\"epoch\",                       # 每个 epoch 保存一次\n",
+    "    learning_rate=2e-5,                          # 学习率\n",
+    "    per_device_train_batch_size=8,               # 每设备的批量大小\n",
+    "    per_device_eval_batch_size=8,                # 每设备评估的批量大小\n",
+    "    num_train_epochs=3,                          # 训练轮数\n",
+    "    weight_decay=0.01,                           # 权重衰减\n",
+    "    logging_dir=\"./logs\",                        # 日志路径\n",
+    "    fp16=True,                                   # 启用混合精度训练\n",
+    "    save_total_limit=2,                          # 保留最多两个检查点\n",
+    "    load_best_model_at_end=True,                 # 加载最佳模型\n",
+    "    metric_for_best_model=\"accuracy\",            # 根据准确率选择最佳模型\n",
+    "    greater_is_better=True,\n",
+    ")\n",
+    "\n",
+    "# **8. 定义 Trainer**\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=train_dataset,\n",
+    "    eval_dataset=test_dataset,\n",
+    "    tokenizer=tokenizer,\n",
+    "    compute_metrics=compute_metrics,\n",
+    ")\n",
+    "\n",
+    "# **9. 开始训练**\n",
+    "trainer.train()\n",
+    "\n",
+    "# **10. 保存模型**\n",
+    "model.save_pretrained(\"./gpt2_lora_text_classification\")\n",
+    "tokenizer.save_pretrained(\"./gpt2_lora_text_classification\")\n",
+    "\n",
+    "print(\"训练完成，模型已保存至 ./gpt2_lora_text_classification\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49a60fed-3a7d-4608-98b1-b4e313b94dbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+    "from peft import PeftModel\n",
+    "\n",
+    "# 加载分词器\n",
+    "model_path = \"./gpt2_lora_text_classification\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
+    "\n",
+    "# 加载微调后的 PEFT 模型\n",
+    "base_model = AutoModelForSequenceClassification.from_pretrained(\"gpt2\", num_labels=2)\n",
+    "model = PeftModel.from_pretrained(base_model, model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c0d8f02-c3dc-4961-8b3a-50eefc5f9448",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "\n",
+    "def predict(texts, model, tokenizer):\n",
+    "    \"\"\"\n",
+    "    使用微调后的 PEFT 模型进行推理。\n",
+    "    \n",
+    "    Args:\n",
+    "        texts (list of str): 待分类的文本列表。\n",
+    "        model (PeftModel): 微调后的模型。\n",
+    "        tokenizer (AutoTokenizer): 分词器。\n",
+    "    \n",
+    "    Returns:\n",
+    "        list of dict: 每个文本的预测结果，包括 logits 和预测的类别标签。\n",
+    "    \"\"\"\n",
+    "    # 对输入文本进行分词和编码\n",
+    "    inputs = tokenizer(\n",
+    "        texts,\n",
+    "        padding=True,\n",
+    "        truncation=True,\n",
+    "        max_length=512,\n",
+    "        return_tensors=\"pt\"\n",
+    "    )\n",
+    "    \n",
+    "    # 将输入数据移动到模型的设备上（CPU/GPU）\n",
+    "    inputs = {key: value.to(model.device) for key, value in inputs.items()}\n",
+    "    \n",
+    "    # 模型推理\n",
+    "    model.eval()\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "    \n",
+    "    # 获取 logits 并计算预测类别\n",
+    "    logits = outputs.logits\n",
+    "    probs = torch.nn.functional.softmax(logits, dim=-1)\n",
+    "    predictions = torch.argmax(probs, dim=-1)\n",
+    "    \n",
+    "    # 返回每个文本的预测结果\n",
+    "    results = [\n",
+    "        {\"text\": text, \"logits\": logit.tolist(), \"predicted_class\": int(pred)}\n",
+    "        for text, logit, pred in zip(texts, logits, predictions)\n",
+    "    ]\n",
+    "    return results\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9c0cfe65-f4f3-4274-a4f4-1ac13725b15a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Text: This movie was fantastic! I loved every part of it.\n",
+    "Predicted Class: 1\n",
+    "Logits: [-2.345, 3.567]\n",
+    "\n",
+    "Text: The plot was terrible and the acting was worse.\n",
+    "Predicted Class: 0\n",
+    "Logits: [4.123, -1.234]\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/6-llama-continue-train.ipynb

@@ -0,0 +1,491 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1e6d4978-4f0f-4268-aa23-d864857bd6c8",
+   "metadata": {},
+   "source": [
+    "# 4.6 基于llama的基因大模型持续预训练"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c201732-e736-463c-8446-637bf517479f",
+   "metadata": {},
+   "source": [
+    "LLaMA（**Large Language Model Meta AI**）是由 Meta（Facebook）开发的一系列大型语言模型，专注于提供高性能和高效的大语言模型，面向学术研究和开发社区。LLaMA 系列主要强调训练效率、模型性能和对计算资源的高效利用，是 GPT 系列模型的有力竞争者之一。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. LLaMA 模型概述**\n",
+    "\n",
+    "#### **1.1 LLaMA 1**\n",
+    "- **发布**：2023 年 2 月。\n",
+    "- **模型参数规模**：\n",
+    "  - 7B（70 亿）\n",
+    "  - 13B（130 亿）\n",
+    "  - 33B（330 亿）\n",
+    "  - 65B（650 亿）\n",
+    "- **特点**：\n",
+    "  - 专注于效率：与 GPT-3 等模型相比，LLaMA 在相同的训练成本下实现了更高的性能。\n",
+    "  - 针对研究开放：提供预训练模型权重供研究使用。\n",
+    "  - 使用高质量的数据：模型训练使用大量从网络中筛选的高质量文本数据，包括维基百科、书籍和其他高质量来源。\n",
+    "- **性能**：\n",
+    "  - 在许多 NLP 任务中，LLaMA 的性能超过 GPT-3 和其他同类模型。\n",
+    "  - 参数规模较小的版本（如 LLaMA-13B）性能可与 GPT-3（175B 参数）媲美。\n",
+    "\n",
+    "#### **1.2 LLaMA 2**\n",
+    "- **发布**：2023 年 7 月。\n",
+    "- **改进**：\n",
+    "  - 增强的训练数据：相比 LLaMA 1，使用了更多的高质量数据。\n",
+    "  - 引入微调版本：发布了开箱即用的对话模型（LLaMA 2-Chat）。\n",
+    "  - 更好的开源支持：LLaMA 2 在商业用途上比 LLaMA 1 更加开放。\n",
+    "- **模型参数规模**：\n",
+    "  - 7B（70 亿）\n",
+    "  - 13B（130 亿）\n",
+    "  - 70B（700 亿）\n",
+    "- **性能**：\n",
+    "  - LLaMA 2 的性能相比 LLaMA 1 有显著提升。\n",
+    "  - LLaMA 2-Chat 在对话任务中的表现优于许多现有开源模型。\n",
+    "  - 在多个标准基准（如 MMLU）上超过 GPT-4 和 Claude 的开源实现。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. LLaMA 的关键技术特点**\n",
+    "\n",
+    "#### **2.1 高效的架构设计**\n",
+    "- 基于 Transformer 架构。\n",
+    "- 针对训练效率和推理速度进行了优化，适合研究和开发。\n",
+    "\n",
+    "#### **2.2 模型压缩**\n",
+    "- 提供更小的参数规模（如 7B 和 13B），以便在更低的计算资源上运行。\n",
+    "- 在性能与参数量之间实现了很好的平衡。\n",
+    "\n",
+    "#### **2.3 训练数据**\n",
+    "- 使用从互联网中提取的高质量数据，注重数据清洗和筛选，避免低质量文本对模型的负面影响。\n",
+    "\n",
+    "#### **2.4 微调能力**\n",
+    "- 支持指令微调（Instruction Tuning）和 RLHF（基于人类反馈的强化学习），特别是在 LLaMA 2-Chat 模型中表现优异。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. LLaMA 的性能对比**\n",
+    "\n",
+    "#### **与 GPT-3 比较**\n",
+    "- LLaMA 1-13B 参数模型在许多任务上的性能接近 GPT-3-175B。\n",
+    "- LLaMA 2-70B 在多个任务上超过 GPT-3。\n",
+    "\n",
+    "#### **与其他开源模型比较**\n",
+    "- LLaMA 2 在许多基准测试中优于其他开源模型（如 Falcon 和 MPT）。\n",
+    "- LLaMA 2-Chat 提供了与 ChatGPT 类似的对话能力，适用于对话任务。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 应用场景**\n",
+    "\n",
+    "1. **研究**：\n",
+    "   - 开源权重适合学术研究，推动了对大语言模型的进一步探索。\n",
+    "\n",
+    "2. **对话系统**：\n",
+    "   - LLaMA 2-Chat 专为对话任务设计，适合开发智能客服、聊天机器人等应用。\n",
+    "\n",
+    "3. **生成任务**：\n",
+    "   - 支持文本生成、补全、摘要等任务。\n",
+    "\n",
+    "4. **微调与定制**：\n",
+    "   - 可以基于特定领域数据进行微调，如医学、法律、教育等领域的专用模型。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 开源与获取方式**\n",
+    "\n",
+    "#### **1. 开源**\n",
+    "- LLaMA 1：需要申请权限才能获得模型权重。\n",
+    "- LLaMA 2：更加开放，允许商业用途，模型和权重可以通过 Meta 的合作平台获取（如 Hugging Face 和 AWS）。\n",
+    "\n",
+    "#### **2. 下载与使用**\n",
+    "使用 Hugging Face 加载模型：\n",
+    "```python\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "model_name = \"meta-llama/Llama-2-7b-hf\"  # 替换为具体模型\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 使用模型生成文本\n",
+    "inputs = tokenizer(\"Hello, how are you?\", return_tensors=\"pt\")\n",
+    "outputs = model.generate(**inputs, max_length=50)\n",
+    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 总结**\n",
+    "\n",
+    "#### **优势**\n",
+    "- **高性能**：在多个基准任务上表现出色。\n",
+    "- **高效训练**：小参数模型能与大模型媲美。\n",
+    "- **开放性**：LLaMA 2 提供了较为开放的商用许可。\n",
+    "\n",
+    "#### **局限**\n",
+    "- 模型需要高质量数据和强大算力训练，对推理设备也有一定要求。\n",
+    "\n",
+    "LLaMA 系列以其高效和开放的特点，为大模型研究和应用带来了强大动力，是当前大语言模型生态的重要组成部分。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7fb0d648-f891-47b9-a644-af5263fa9718",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8b3c9ebb-213b-4dc4-a712-5a819fea3197",
+   "metadata": {},
+   "source": [
+    "**大模型的持续预训练**（Continual Pretraining of Large Models）是指在基础预训练模型（如 GPT、BERT 等）的基础上，通过引入新的数据或特定领域的数据继续进行预训练的过程。这一过程旨在让模型在特定场景或任务中表现更好，同时保留其通用能力。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 持续预训练的概念**\n",
+    "\n",
+    "持续预训练是一种在通用大模型的预训练基础上，进一步优化和适配模型的方法，主要包括以下两种场景：\n",
+    "1. **领域适配**：\n",
+    "   - 将预训练模型在特定领域的数据上继续训练，使其对该领域的语料理解更深刻，例如法律、医学、金融等领域。\n",
+    "2. **性能优化**：\n",
+    "   - 通过引入更多的通用数据或多样化的数据类型，扩展模型的通用能力，提高性能。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 持续预训练的目标**\n",
+    "\n",
+    "1. **提升领域性能**：\n",
+    "   - 在特定领域任务上，模型能够更好地理解特定领域的语言模式和知识。\n",
+    "   \n",
+    "2. **增强模型鲁棒性**：\n",
+    "   - 通过引入新的数据或增强数据多样性，使模型对未见数据表现更稳定。\n",
+    "\n",
+    "3. **优化资源利用**：\n",
+    "   - 通过复用已有的大模型权重，只需训练少量额外步骤，避免从零开始重新训练模型。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 持续预训练的步骤**\n",
+    "\n",
+    "#### **（1）数据准备**\n",
+    "- **领域数据**：针对特定领域（如医学、法律、科技）收集高质量语料。\n",
+    "- **新语料整合**：补充模型未见过的多样化语料。\n",
+    "- **数据清洗**：确保数据无噪声、语言风格一致。\n",
+    "\n",
+    "#### **（2）模型初始化**\n",
+    "- 使用现有的预训练模型作为初始权重，例如 Hugging Face 提供的 GPT-2 或 BERT 模型。\n",
+    "\n",
+    "#### **（3）训练设置**\n",
+    "- **超参数调整**：\n",
+    "  - 通常使用较小的学习率（例如 `1e-5` 或 `2e-5`）以避免破坏已有的知识。\n",
+    "- **训练策略**：\n",
+    "  - 冻结部分参数（如嵌入层或前几层）以保留通用能力，仅调整高层或新加入的部分。\n",
+    "\n",
+    "#### **（4）评估和验证**\n",
+    "- 使用领域任务的数据集对模型进行评估，验证其在目标任务中的改进效果。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 持续预训练的常见方法**\n",
+    "\n",
+    "#### **（1）全量持续预训练**\n",
+    "- 对整个模型的参数进行调整。\n",
+    "- **优点**：适合较大规模的新数据训练，能显著提升领域性能。\n",
+    "- **缺点**：计算资源需求大，可能导致模型过拟合。\n",
+    "\n",
+    "#### **（2）冻结部分参数**\n",
+    "- 冻结低层参数，仅微调高层。\n",
+    "- **优点**：保留通用知识，减少计算开销。\n",
+    "- **缺点**：对领域特定知识的适配可能不足。\n",
+    "\n",
+    "#### **（3）参数高效微调（PEFT）**\n",
+    "- 使用 PEFT 方法（如 LoRA、Adapter）进行预训练：\n",
+    "  - **LoRA**：通过低秩矩阵分解，微调部分关键模块。\n",
+    "  - **Adapter**：在 Transformer 层中插入小型适配模块。\n",
+    "- **优点**：显著减少需要更新的参数量。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 持续预训练的典型应用**\n",
+    "\n",
+    "1. **领域适配**\n",
+    "   - **医学**：将预训练模型在 PubMed 或生物医学数据集上进行持续预训练。\n",
+    "   - **法律**：使用法律文档进一步训练基础模型。\n",
+    "   - **金融**：通过金融新闻、报告语料提升模型在金融领域的表现。\n",
+    "\n",
+    "2. **多语言扩展**\n",
+    "   - 引入多语言语料，扩展模型的多语言能力。\n",
+    "\n",
+    "3. **数据更新**\n",
+    "   - 持续加入新数据（如时事新闻）以适配最新语言模式。\n",
+    "\n",
+    "4. **特殊任务优化**\n",
+    "   - 针对特定任务（如代码生成、对话）引入专用数据进行训练。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 实现持续预训练的代码示例**\n",
+    "\n",
+    "以下示例基于 Hugging Face 实现 GPT-2 的持续预训练：\n",
+    "\n",
+    "```python\n",
+    "from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# 1. 加载预训练模型和分词器\n",
+    "model_name = \"gpt2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 2. 加载新语料数据\n",
+    "dataset = load_dataset(\"text\", data_files={\"train\": \"domain_corpus.txt\"})\n",
+    "\n",
+    "# 3. 数据预处理\n",
+    "def tokenize_function(examples):\n",
+    "    return tokenizer(examples[\"text\"], truncation=True, max_length=1024, padding=\"max_length\")\n",
+    "\n",
+    "tokenized_dataset = dataset.map(tokenize_function, batched=True)\n",
+    "\n",
+    "# 4. 设置训练参数\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2_domain_adapted\",\n",
+    "    overwrite_output_dir=True,\n",
+    "    per_device_train_batch_size=4,\n",
+    "    num_train_epochs=3,\n",
+    "    learning_rate=5e-5,\n",
+    "    save_steps=500,\n",
+    "    save_total_limit=2,\n",
+    "    logging_dir=\"./logs\",\n",
+    "    evaluation_strategy=\"no\",  # 评估策略可以根据需要调整\n",
+    "    fp16=True,  # 混合精度训练\n",
+    ")\n",
+    "\n",
+    "# 5. 定义 Trainer 并启动训练\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_dataset[\"train\"],\n",
+    "    tokenizer=tokenizer,\n",
+    ")\n",
+    "\n",
+    "trainer.train()\n",
+    "\n",
+    "# 6. 保存模型\n",
+    "model.save_pretrained(\"./gpt2_domain_adapted\")\n",
+    "tokenizer.save_pretrained(\"./gpt2_domain_adapted\")\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 持续预训练的挑战**\n",
+    "\n",
+    "1. **灾难性遗忘**：\n",
+    "   - 持续预训练可能导致模型丧失之前学到的知识。\n",
+    "   - **解决方法**：使用少量原始数据进行联合训练。\n",
+    "\n",
+    "2. **计算资源需求**：\n",
+    "   - 需要大量显存和算力，特别是对于大规模模型和数据。\n",
+    "\n",
+    "3. **数据质量和多样性**：\n",
+    "   - 新引入的数据可能包含噪声，影响模型性能。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **8. 持续预训练的优势**\n",
+    "\n",
+    "- 提高特定领域或任务的性能。\n",
+    "- 更高效地利用已有模型权重，避免从头训练。\n",
+    "- 保留原始模型的通用能力，同时增强领域适应性。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "持续预训练是适配领域任务和提升模型性能的重要方法，通过引入新数据或优化模型训练策略，可以让大模型在特定场景中表现更优。配合参数高效微调方法（如 LoRA），还可显著降低计算开销，提升训练效率。这种技术在学术研究、工业应用和前沿领域（如法律、医学等）中均具有广泛价值。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca41ad33-18fb-44da-8f79-0380b5c9dcaa",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3038550c-cc92-45c9-8bb4-46c58688bfc5",
+   "metadata": {},
+   "source": [
+    "## 本节任务\n",
+    "本节任务是基于llama。训练一个能够处理dna和protein蛋白质数据的基础预训练大模型，数据为第一章中的预训练数据，包括英文数据。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b1bd33b8-2e05-4b59-9d8f-c48de194cfd6",
+   "metadata": {},
+   "source": [
+    "## 代码运行\n",
+    "\n",
+    "```\n",
+    "# 复制第一章训练数据,包括dna，protein，还有英文数据，添加英文数据是为了避免遗忘问题\n",
+    "mkdir train_data\n",
+    "cp ../01-data_env/data/*.txt train_data/\n",
+    "\n",
+    "#持续预训练\n",
+    "./run_pt.sh\n",
+    "\n",
+    "#合并模型\n",
+    "./merge_pt_model.sh\n",
+    "\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4960a36c-7529-4db8-b91d-df91245f79d9",
+   "metadata": {},
+   "source": [
+    "## 模型验证"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69b3e97f-a801-4264-a651-a854bcfba9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "339435d9-9379-4b30-ae8b-50feee1ba714",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-merge-hf\")\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d0f154bb-b1ab-4611-a14c-9b403043fd96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-merge-hf\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "792a9f78-1828-4695-9f6e-479a704ea7e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoConfig\n",
+    "# 加载配置\n",
+    "config = AutoConfig.from_pretrained('dnahlm-merge-hf')\n",
+    "config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49021c65-54bb-4a97-a96d-b030cc3dcd13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text='''GCTGACTCTGCCAGGATGGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCTAGTCAGACATAGTCACATAGGCAAGTAAGGGAACCTAAAATTGCTTGGAAT,\n",
+    "KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV,\n",
+    "The primary use of LLaMA is research on large language models, including'''\n",
+    "print(\"Test text:\\n\",text)\n",
+    "print(f\"Tokenized by DNA-LLaMA tokenizer:{tokenizer.tokenize(text)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ebf869c8-866d-4770-8f64-79d671f88663",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from transformers import pipeline\n",
+    "\n",
+    "model_id = \"dnahlm-merge-hf\"\n",
+    "\n",
+    "pipe = pipeline(\n",
+    "    \"text-generation\", \n",
+    "    model=model_id, \n",
+    "    #torch_dtype=torch.bfloat16, \n",
+    "    device_map=\"auto\",\n",
+    ")\n",
+    "\n",
+    "pipe(\"The key to life is\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40a22c70-f1c4-4cd5-a118-2f5db40790e6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe(\"GGAATGAAATTAGGTTGTTTTAATTATAATGTAAAGTCAGTTCT\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aec95d0a-4269-4540-bf14-4ce157b9a194",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipe(\"KCGFVGPMVHLKVHLEADVASSCRSAVIYLTSEEPFEGVLGLRLKEGIAITGCWPRWPDEMDERSAVWRVEPYTRHFGRVLYSFGV\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/7-llama-instruction-ft.ipynb

@@ -0,0 +1,624 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "963e9ae0-ac68-44be-8c7d-fb9842784362",
+   "metadata": {},
+   "source": [
+    "# 4.7 基于llama的基因大模型指令微调"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c844103d-4e27-41b9-9bf1-c6a577846ab6",
+   "metadata": {},
+   "source": [
+    "### **大模型的指令微调（Instruction Fine-Tuning）**\n",
+    "\n",
+    "指令微调是指通过对大语言模型（如 GPT、T5、LLaMA 等）进行微调，使其能够更好地理解和执行人类以指令形式表达的任务。这种技术是大模型适配实际应用和增强用户交互能力的关键手段。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 指令微调的核心概念**\n",
+    "\n",
+    "指令微调的目标是通过在包含指令的专用数据集上进行微调，让模型能够：\n",
+    "1. 理解用户的任务需求（以自然语言表达的指令形式）。\n",
+    "2. 根据指令内容生成符合预期的高质量响应。\n",
+    "3. 适应多任务场景，减少特定任务的单独训练需求。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 指令微调的关键特点**\n",
+    "\n",
+    "1. **多任务统一**：\n",
+    "   - 不需要针对每个任务单独微调，而是通过指令微调使模型能适应多种任务。\n",
+    "   \n",
+    "2. **自然语言交互**：\n",
+    "   - 用户可以用自然语言指令与模型交互，无需提供特定格式的输入。\n",
+    "\n",
+    "3. **泛化能力**：\n",
+    "   - 微调后的模型能够对未见过的任务产生合理的推断和响应。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 数据集的构建与使用**\n",
+    "\n",
+    "#### **（1）指令微调数据集的特点**\n",
+    "- 数据通常包含以下三部分：\n",
+    "  1. **指令（Instruction）**：任务描述或问题，例如“将以下文本翻译为法语”。\n",
+    "  2. **输入（Input）**：任务相关的上下文或数据，可以为空。\n",
+    "  3. **输出（Output）**：模型期望生成的结果。\n",
+    "\n",
+    "#### **（2）常用指令微调数据集**\n",
+    "- **FLAN**：包含多个 NLP 任务的指令数据集，用于 T5 等模型的微调。\n",
+    "- **OpenAI 提供的指令数据**：如 GPT 系列的 ChatGPT 调优数据集。\n",
+    "- **InstructGPT 数据**：通过人类标注的多任务指令数据，用于模型优化。\n",
+    "- **Self-Instruct**：通过模型自生成指令和回答，进一步扩展训练数据。\n",
+    "\n",
+    "#### **（3）构建自己的数据集**\n",
+    "- 如果需要特定领域的指令微调，可以自行构建数据集：\n",
+    "  - 收集任务需求和示例。\n",
+    "  - 设计多样化的指令。\n",
+    "  - 使用专家标注或模型辅助生成高质量答案。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 微调的步骤**\n",
+    "\n",
+    "#### **（1）加载基础模型**\n",
+    "从 Hugging Face 或其他框架加载预训练的大语言模型，例如 GPT-2、T5、LLaMA。\n",
+    "\n",
+    "#### **（2）准备数据集**\n",
+    "将指令微调数据集格式化为：\n",
+    "```python\n",
+    "{\n",
+    "    \"instruction\": \"Translate the following text to French\",\n",
+    "    \"input\": \"Hello, how are you?\",\n",
+    "    \"output\": \"Bonjour, comment ça va?\"\n",
+    "}\n",
+    "```\n",
+    "\n",
+    "#### **（3）定义微调方法**\n",
+    "使用 `Trainer` 或分布式框架（如 DeepSpeed、Accelerate）进行微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 示例代码：指令微调实现**\n",
+    "\n",
+    "以下是基于 Hugging Face 的指令微调代码示例：\n",
+    "\n",
+    "```python\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# 1. 加载预训练模型和分词器\n",
+    "model_name = \"gpt2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
+    "\n",
+    "# 2. 加载指令微调数据集\n",
+    "# 数据格式应包含 instruction, input, output 字段\n",
+    "dataset = load_dataset(\"path/to/instruction_dataset\")\n",
+    "\n",
+    "# 3. 数据预处理\n",
+    "def preprocess_function(example):\n",
+    "    # 将指令和输入拼接成完整的提示\n",
+    "    prompt = example[\"instruction\"]\n",
+    "    if example[\"input\"]:\n",
+    "        prompt += f\"\\n{example['input']}\"\n",
+    "    labels = example[\"output\"]\n",
+    "    tokenized = tokenizer(prompt, truncation=True, max_length=512, padding=\"max_length\")\n",
+    "    with tokenizer.as_target_tokenizer():\n",
+    "        tokenized_labels = tokenizer(labels, truncation=True, max_length=512, padding=\"max_length\")\n",
+    "    tokenized[\"labels\"] = tokenized_labels[\"input_ids\"]\n",
+    "    return tokenized\n",
+    "\n",
+    "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
+    "\n",
+    "# 4. 设置训练参数\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./instruction_finetuned_model\",\n",
+    "    per_device_train_batch_size=4,\n",
+    "    num_train_epochs=3,\n",
+    "    evaluation_strategy=\"epoch\",\n",
+    "    save_strategy=\"epoch\",\n",
+    "    learning_rate=5e-5,\n",
+    "    weight_decay=0.01,\n",
+    "    logging_dir=\"./logs\",\n",
+    "    fp16=True,\n",
+    ")\n",
+    "\n",
+    "# 5. 定义 Trainer\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_datasets[\"train\"],\n",
+    "    eval_dataset=tokenized_datasets[\"test\"],\n",
+    "    tokenizer=tokenizer,\n",
+    ")\n",
+    "\n",
+    "# 6. 开始训练\n",
+    "trainer.train()\n",
+    "\n",
+    "# 7. 保存模型\n",
+    "model.save_pretrained(\"./instruction_finetuned_model\")\n",
+    "tokenizer.save_pretrained(\"./instruction_finetuned_model\")\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 指令微调的挑战**\n",
+    "\n",
+    "1. **数据质量**：\n",
+    "   - 低质量或噪声数据可能导致模型生成结果不符合指令。\n",
+    "\n",
+    "2. **指令覆盖范围**：\n",
+    "   - 数据集指令种类不足会限制模型的泛化能力。\n",
+    "\n",
+    "3. **计算资源需求**：\n",
+    "   - 大模型的微调需要高性能 GPU 和大容量存储。\n",
+    "\n",
+    "4. **灾难性遗忘**：\n",
+    "   - 微调过程中可能导致模型丧失部分原始能力。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **7. 指令微调的应用场景**\n",
+    "\n",
+    "1. **多任务问答**：\n",
+    "   - 适配多任务场景，支持翻译、总结、推理等功能。\n",
+    "\n",
+    "2. **特定领域优化**：\n",
+    "   - 在法律、医疗等特定领域的任务指令上进行微调。\n",
+    "\n",
+    "3. **用户交互优化**：\n",
+    "   - 提升模型对自然语言指令的理解和响应能力。\n",
+    "\n",
+    "4. **开放式对话生成**：\n",
+    "   - 优化模型在对话场景下的表现，例如 ChatGPT 的微调。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "指令微调通过在特定格式的数据集上进一步训练大模型，使其能够更好地理解和执行用户的自然语言指令。这种方法适合多任务场景，并能提升模型的交互能力和领域适应性。借助高质量的指令数据集和高效的微调技术，大模型在实际应用中的表现可以得到显著提升。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7be8b814-42f6-4fb6-bf4b-ae23292030f6",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9bed0ae-337d-49af-85f0-c8e6263d78db",
+   "metadata": {},
+   "source": [
+    "**大模型的持续预训练**和**指令微调**是两种针对大模型的后续优化策略，虽然它们的目标都是提升模型性能，但在应用场景、方法和效果等方面有明显区别。以下是它们的对比分析：\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **1. 概念与目标**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **定义**               | 在通用预训练模型上，使用新的大规模语料（通用或领域特定数据）进行进一步预训练。                      | 在包含指令任务的数据集上对大模型进行微调，以提升模型对人类指令的理解和执行能力。                              |\n",
+    "| **目标**               | 提升模型的通用能力或适应特定领域的语言理解与生成能力。                                              | 提高模型对多任务指令的泛化能力，让模型更好地理解和执行自然语言表达的具体任务。                                |\n",
+    "| **典型应用**           | 领域适配（医学、法律、金融）、性能优化、跨语言适配等。                                              | 多任务问答、开放式对话生成、翻译、推理等需要用户直接交互的场景。                                              |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **2. 数据使用**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **数据类型**           | 通用语料（如新闻、社交媒体文本）或领域特定语料（如 PubMed、法律文档、金融报告）。                    | 任务指令数据集，包括指令（Instruction）、输入（Input）和输出（Output）。                                   |\n",
+    "| **数据构建**           | 通常需要清洗和去重大规模语料数据，避免与原始预训练数据重叠。                                         | 通常由人工标注或模型生成的指令数据构成，例如 FLAN、InstructGPT 数据集。                                     |\n",
+    "| **多样性要求**         | 数据应覆盖尽可能广的领域或目标领域的多种场景，以提升模型在这些场景的表现。                                | 数据需要覆盖多种任务类型（如翻译、分类、摘要）和丰富的指令表达形式，以提高模型对多任务的适配能力。                 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **3. 方法与技术**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **主要技术**           | 继续使用自监督学习目标（如语言建模、掩码预测）进行训练。                                            | 使用监督学习，通常以任务输入和目标输出对为数据，通过微调适配特定任务需求。                                  |\n",
+    "| **模型调整**           | - 可选择全量参数更新或冻结部分参数。<br>- 可结合参数高效微调技术（如 LoRA、Adapter）。                  | - 通常使用监督训练方式，可能结合参数高效微调技术（如 LoRA）。                                               |\n",
+    "| **学习率**             | 通常使用较小的学习率（如 `1e-5` 或更小），以防止破坏原始权重。                                         | 同样使用较小的学习率，但任务指令微调可能需要更高的关注任务特定的标签对准。                                     |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **4. 模型能力与效果**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **提升的能力**         | - 对领域特定语言模式和知识的适配性提升显著。<br>- 对未见过的通用场景生成能力增强（扩展模型知识广度）。       | - 显著提升模型对指令理解的能力，尤其是自然语言表达的任务需求。<br>- 对多任务和零样本任务的泛化能力有较大提升。  |\n",
+    "| **局限性**             | - 对具体任务的直接适配能力较弱，可能需要额外的任务微调。<br>- 数据选择不当可能导致灾难性遗忘。                 | - 依赖高质量的指令数据集，数据质量不高会导致模型生成结果不稳定。<br>- 对通用能力的提升有限。                    |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **5. 应用场景与示例**\n",
+    "\n",
+    "| **特性**               | **持续预训练**                                                                                  | **指令微调**                                                                                              |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|\n",
+    "| **典型应用场景**       | - 医学文献总结（通过 PubMed 语料持续预训练）。<br>- 法律条文分析（通过法律文档进一步训练）。<br>- 增强多语言生成能力（跨语言语料）。 | - ChatGPT 的多任务对话生成。<br>- 翻译、摘要、问答等用户交互任务的泛化处理。                                 |\n",
+    "| **实际示例**           | - BioBERT：在 BERT 基础上使用生物医学语料持续预训练的模型。<br>- FinBERT：针对��融领域持续预训练的语言模型。 | - InstructGPT：在 GPT-3 基础上进行指令微调，用于多任务用户交互。<br>- FLAN-T5：通过 FLAN 数据集进行指令微调。 |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **6. 持续预训练与指令微调的结合**\n",
+    "\n",
+    "持续预训练和指令微调可以结合使用，形成一个从领域适配到任务适配的完整流程：\n",
+    "1. **持续预训练**：\n",
+    "   - 先在领域特定数据（如医学、法律、金融语料）上进行持续预训练，获取领域知识。\n",
+    "2. **指令微调**：\n",
+    "   - 再利用多任务指令数据集对模型微调，使其能够高效执行领域内的多样化任务。\n",
+    "\n",
+    "这种结合方式特别适用于需要领域知识和任务适配的场景，例如医学问答系统或金融文本分析。\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### **总结**\n",
+    "\n",
+    "| **维度**               | **持续预训练**                      | **指令微调**                     |\n",
+    "|------------------------|-------------------------------------|----------------------------------|\n",
+    "| **目标**               | 增强通用能力或适配特定领域。          | 提升对任务指令的理解和执行能力。    |\n",
+    "| **数据集**             | 通用或领域语料。                    | 指令数据集，包含输入和输出对。      |\n",
+    "| **方法**               | 自监督学习，扩展语言建模能力。         | 监督学习，强化任务适配能力。        |\n",
+    "| **适用场景**           | 领域特定任务（如医学、法律）。         | 多任务交互（如问答、对话生成）。     |\n",
+    "| **局限性**             | 对具体任务适配较弱。                 | 通用能力提升有限，依赖数据质量。     |\n",
+    "\n",
+    "两者各有侧重，且在许多场景下可以结合使用，形成一个强大的任务和领域适配框架。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f97a705a-b946-4dc1-a173-a9df033d6f2b",
+   "metadata": {},
+   "source": [
+    "## 本节任务\n",
+    "本节任务是基于上一节预训练的llama生物大模型。对一些生物学任务进行微调，包含了多个不同类型的分类问题和多序列交换问题。具体可见sft_data下的数据。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9782db62-95bd-40a6-9759-966b9a0b362e",
+   "metadata": {},
+   "source": [
+    "## 代码运行\n",
+    "\n",
+    "```\n",
+    "\n",
+    "#微调\n",
+    "./run_sft.sh\n",
+    "\n",
+    "#合并模型\n",
+    "./merge_sft_model.sh\n",
+    "\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "182b82c4-d484-4c15-a600-03c3b51367ec",
+   "metadata": {},
+   "source": [
+    "## 模型验证"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5aa3d240-44e1-4811-8f61-d6ff2500a798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "# 设置环境变量, autodl一般区域\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17bdb69d-3f0f-465e-bd60-2047a088e264",
+   "metadata": {},
+   "source": [
+    "如果您不确定模型中有哪些模块可以微调，可以打印模型结构："
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "054a2956-9045-4ad5-a878-1bfc84ad4ed8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63c8bf16-9576-41bc-b27c-c92ba4289cf4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
+    "dna_ft_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95928da3-ca64-4a17-80f4-945da395702c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a3e65bcd-85ce-4261-8ba6-7665c4ec60e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #dnagpt/dnahlm-llama-7b-sft-v0\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d3fe49b-f48f-42b2-bc97-028e443111e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c54df9fe-86c4-4963-b313-b438894bf9dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#构建提示词\n",
+    "def format_input(entry):\n",
+    "    instruction_text = (\n",
+    "        f\"Below is an instruction that describes a task. \"\n",
+    "        f\"Write a response that appropriately completes the request.\"\n",
+    "        f\"\\n\\n### Instruction:\\n{entry['instruction']}\"\n",
+    "    )\n",
+    "\n",
+    "    input_text = f\"\\n\\n### Input:\\n{entry['input']}\" if entry[\"input\"] else \"\"\n",
+    "\n",
+    "    return instruction_text + input_text + \"\\n\\n### Response:\\n\"\n",
+    "\n",
+    "#构建提示词\n",
+    "def build_prompt(entry):\n",
+    "\n",
+    "    input_data = format_input(entry)\n",
+    "\n",
+    "    desired_response = entry['output']\n",
+    "\n",
+    "    return input_data + desired_response\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee540cfb-1f6e-4e02-a3bc-c814e43685cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "example = data[\"test\"][0]\n",
+    "example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ee35528-7b3f-4e60-b88b-1bc3e950012b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = build_prompt(example)\n",
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8aa6f38f-3bcc-4566-8a66-a541db91e031",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.tokenize(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11875339-4901-4912-86e5-afe8c74921d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=1000):\n",
+    "  # Tokenize\n",
+    "  input_ids = tokenizer.encode(\n",
+    "          text,\n",
+    "          return_tensors=\"pt\",\n",
+    "          truncation=True,\n",
+    "          max_length=max_input_tokens\n",
+    "          # return_attention_mask=True,\n",
+    "  )\n",
+    "\n",
+    "  # Generate\n",
+    "  device = model.device\n",
+    "  generated_tokens_with_prompt = model.generate(\n",
+    "    input_ids=input_ids.to(device),\n",
+    "    #max_length=max_output_tokens,\n",
+    "    max_new_tokens=8,\n",
+    "    temperature=0.01  # 控制生成的多样性\n",
+    "  )\n",
+    "\n",
+    "  # Decode\n",
+    "  generated_text_with_prompt = tokenizer.decode(generated_tokens_with_prompt[0], skip_special_tokens=True)\n",
+    "  generated_text_answer = generated_text_with_prompt[len(text):]\n",
+    "\n",
+    "\n",
+    "  return generated_text_answer\n",
+    "\n",
+    "# 如果需要进一步清理\n",
+    "def clean_generated_text(text):\n",
+    "    # 去除 'Ġ' 符号并替换为空格\n",
+    "    text = text.replace('Ġ', ' ')\n",
+    "    # 去除多余的空格\n",
+    "    text = ' '.join(text.split())\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b02644a-8b24-45aa-b22d-0f7ce2270dd9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_text = format_input(data[\"test\"][0])\n",
+    "\n",
+    "print(\"input (test):\", input_text)\n",
+    "\n",
+    "print(\"real answer:\", data[\"test\"][0][\"output\"])\n",
+    "\n",
+    "print(\"--------------------------\\n\")\n",
+    "\n",
+    "print(\"model's answer: \\n\")\n",
+    "print(inference(input_text, model, tokenizer))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e2df1569-7f70-46ee-b93f-cbd879e32e14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_data = data[\"test\"].shuffle(seed=199).select(range(100))\n",
+    "\n",
+    "data_list = []\n",
+    "\n",
+    "for entry in test_data:\n",
+    "    input_text = format_input(entry)\n",
+    "    #print(input_text)\n",
+    "    response_text = inference(input_text, model, tokenizer)\n",
+    "    #print(response_text)\n",
+    "    data = {\n",
+    "        \"instruction\":entry[\"instruction\"],\n",
+    "         \"input\":entry[\"input\"],\n",
+    "         \"output\":entry[\"output\"],\n",
+    "        \"model_response\":response_text\n",
+    "    }\n",
+    "\n",
+    "    data_list.append(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c6e47cb-1b64-4690-a51d-f1816b82f15f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 定义输出文件路径\n",
+    "output_file = 'llama-sft-2.json'\n",
+    "\n",
+    "# 将 Dataset 对象导出为 JSON 文件\n",
+    "# test_data.to_json(output_file)\n",
+    "with open(output_file, \"w\") as file:\n",
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68831e19-5a99-46d8-9f40-e8bf6957dbfc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "\n",
+    "\n",
+    "with open(output_file, \"r\") as file:\n",
+    "    test_data = json.load(file)\n",
+    "\n",
+    "all_num = len(test_data)\n",
+    "right_sum = 0\n",
+    "same_sum = 0\n",
+    "for item in test_data:\n",
+    "    output = item[\"output\"]\n",
+    "    #output = \" \".join(tokenizer.tokenize(output))\n",
+    "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
+    "    if model_response == output: #same it\n",
+    "        same_sum = same_sum + 1\n",
+    "        \n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
+    "\n",
+    "\n",
+    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/gene_bpe_seg.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:433c302f3c6642f4400e95e5143e08d3cf1a102fd34e4143b2c837550b13e8a6
+size 1102702

04-gene-sft/llama_sft_test.ipynb

@@ -0,0 +1,1627 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "81a2413e-8629-4016-aace-17d2f757f726",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "https://hf-mirror.com\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "# 设置环境变量\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
+    "\n",
+    "# 打印环境变量以确认设置成功\n",
+    "print(os.environ.get('HF_ENDPOINT'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "89e2d33a-6d84-4ef3-b44e-daa57ac81e58",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-11-24 11:21:51.020375: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
+      "2024-11-24 11:21:51.036615: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
+      "2024-11-24 11:21:51.053557: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
+      "2024-11-24 11:21:51.058466: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
+      "2024-11-24 11:21:51.071840: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX512_FP16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "2024-11-24 11:21:51.923693: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoConfig,AutoModel\n",
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "from transformers import  AutoConfig, AutoModelForCausalLM,LlamaForCausalLM,LlamaTokenizer\n",
+    "from tokenizers import Tokenizer\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "68fc5c44-b444-402e-aaf2-0ba4e2000e42",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 19839\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "dna_ft_dataset = load_dataset('json', data_files='val_data.json')\n",
+    "dna_ft_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4ab4fd3e-5b59-470e-9b46-f0ffd7b9d1aa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 1983\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['instruction', 'input', 'output'],\n",
+       "        num_rows: 17856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = dna_ft_dataset[\"train\"].train_test_split(train_size=0.1, seed=42)\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "85ca97f5-6864-4d6f-944a-182ed1fa2f00",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = LlamaTokenizer.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #dnagpt/dnahlm-llama-7b-sft-v0\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e904c0b2-bf21-4036-b510-8e57177c1767",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "99ce92d0373a498d929bed42f770ed16",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "LlamaForCausalLM(\n",
+       "  (model): LlamaModel(\n",
+       "    (embed_tokens): Embedding(61973, 4096, padding_idx=0)\n",
+       "    (layers): ModuleList(\n",
+       "      (0-31): 32 x LlamaDecoderLayer(\n",
+       "        (self_attn): LlamaSdpaAttention(\n",
+       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
+       "          (rotary_emb): LlamaRotaryEmbedding()\n",
+       "        )\n",
+       "        (mlp): LlamaMLP(\n",
+       "          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
+       "          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
+       "          (act_fn): SiLU()\n",
+       "        )\n",
+       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "      )\n",
+       "    )\n",
+       "    (norm): LlamaRMSNorm((4096,), eps=1e-06)\n",
+       "    (rotary_emb): LlamaRotaryEmbedding()\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=4096, out_features=61973, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = LlamaForCausalLM.from_pretrained(\"dnahlm-llama-7b-sft-v0\") #continue pretrain\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "5b361c5c-c43f-4ed9-a5c7-c72403cd7a0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#构建提示词\n",
+    "def format_input(entry):\n",
+    "    instruction_text = (\n",
+    "        f\"Below is an instruction that describes a task. \"\n",
+    "        f\"Write a response that appropriately completes the request.\"\n",
+    "        f\"\\n\\n### Instruction:\\n{entry['instruction']}\"\n",
+    "    )\n",
+    "\n",
+    "    input_text = f\"\\n\\n### Input:\\n{entry['input']}\" if entry[\"input\"] else \"\"\n",
+    "\n",
+    "    return instruction_text + input_text + \"\\n\\n### Response:\\n\"\n",
+    "\n",
+    "#构建提示词\n",
+    "def build_prompt(entry):\n",
+    "\n",
+    "    input_data = format_input(entry)\n",
+    "\n",
+    "    desired_response = entry['output']\n",
+    "\n",
+    "    return input_data + desired_response\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "ed031a26-d79e-4f50-85d1-169ebd409c6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'instruction': 'Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.',\n",
+       " 'input': 'CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC',\n",
+       " 'output': 'promoter'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "example = data[\"test\"][0]\n",
+    "example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "31bd4bb5-86a6-4046-b510-492b0548323b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "promoter\n"
+     ]
+    }
+   ],
+   "source": [
+    "prompt = build_prompt(example)\n",
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "ed0b5b8b-916c-499b-a6da-f1124b9add1c",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['▁Below',\n",
+       " '▁is',\n",
+       " '▁an',\n",
+       " '▁instruction',\n",
+       " '▁that',\n",
+       " '▁describes',\n",
+       " '▁a',\n",
+       " '▁task',\n",
+       " '.',\n",
+       " '▁Write',\n",
+       " '▁a',\n",
+       " '▁response',\n",
+       " '▁that',\n",
+       " '▁appropri',\n",
+       " 'ately',\n",
+       " '▁comple',\n",
+       " 'tes',\n",
+       " '▁the',\n",
+       " '▁request',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Inst',\n",
+       " 'ruction',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'Det',\n",
+       " 'erm',\n",
+       " 'ine',\n",
+       " '▁core',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '▁detection',\n",
+       " '▁of',\n",
+       " '▁following',\n",
+       " '▁d',\n",
+       " 'na',\n",
+       " '▁sequence',\n",
+       " ',',\n",
+       " '▁The',\n",
+       " '▁result',\n",
+       " '▁will',\n",
+       " '▁be',\n",
+       " '▁one',\n",
+       " '▁of',\n",
+       " '▁the',\n",
+       " '▁following',\n",
+       " ':',\n",
+       " '▁Non',\n",
+       " '-',\n",
+       " 'prom',\n",
+       " 'oter',\n",
+       " ',',\n",
+       " '▁prom',\n",
+       " 'oter',\n",
+       " '.',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Input',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'CCGTG',\n",
+       " 'C',\n",
+       " 'GAC',\n",
+       " 'CGGAA',\n",
+       " 'GTG',\n",
+       " 'GGGC',\n",
+       " 'GGC',\n",
+       " 'GAC',\n",
+       " 'CCCGGAA',\n",
+       " 'GTCC',\n",
+       " 'CCGCC',\n",
+       " 'GGGTG',\n",
+       " 'CA',\n",
+       " 'GCT',\n",
+       " 'TG',\n",
+       " 'GTC',\n",
+       " 'GGT',\n",
+       " 'TC',\n",
+       " 'GATCGCC',\n",
+       " '<0x0A>',\n",
+       " '<0x0A>',\n",
+       " '##',\n",
+       " '#',\n",
+       " '▁Response',\n",
+       " ':',\n",
+       " '<0x0A>',\n",
+       " 'prom',\n",
+       " 'oter']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.tokenize(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "f0449aee-1ac6-4db5-873f-afdfb0fc9691",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=1000):\n",
+    "  # Tokenize\n",
+    "  input_ids = tokenizer.encode(\n",
+    "          text,\n",
+    "          return_tensors=\"pt\",\n",
+    "          truncation=True,\n",
+    "          max_length=max_input_tokens\n",
+    "          # return_attention_mask=True,\n",
+    "  )\n",
+    "\n",
+    "  # Generate\n",
+    "  device = model.device\n",
+    "  generated_tokens_with_prompt = model.generate(\n",
+    "    input_ids=input_ids.to(device),\n",
+    "    #max_length=max_output_tokens,\n",
+    "    max_new_tokens=8,\n",
+    "    temperature=0.01  # 控制生成的多样性\n",
+    "  )\n",
+    "\n",
+    "  # Decode\n",
+    "  generated_text_with_prompt = tokenizer.decode(generated_tokens_with_prompt[0], skip_special_tokens=True)\n",
+    "  generated_text_answer = generated_text_with_prompt[len(text):]\n",
+    "\n",
+    "\n",
+    "  return generated_text_answer\n",
+    "\n",
+    "# 如果需要进一步清理\n",
+    "def clean_generated_text(text):\n",
+    "    # 去除 'Ġ' 符号并替换为空格\n",
+    "    text = text.replace('Ġ', ' ')\n",
+    "    # 去除多余的空格\n",
+    "    text = ' '.join(text.split())\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "e9041426-eb59-4314-82dd-7b6d6d477783",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input (test): Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+      "\n",
+      "### Instruction:\n",
+      "Determine core promoter detection of following dna sequence, The result will be one of the following: Non-promoter, promoter.\n",
+      "\n",
+      "### Input:\n",
+      "CCGTGCGACCGGAAGTGGGGCGGCGACCCCGGAAGTCCCCGCCGGGTGCAGCTTGGTCGGTTCGATCGCC\n",
+      "\n",
+      "### Response:\n",
+      "\n",
+      "real answer: promoter\n",
+      "--------------------------\n",
+      "\n",
+      "model's answer: \n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.12/site-packages/transformers/generation/configuration_utils.py:601: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.01` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+      "  warnings.warn(\n",
+      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " Non-promoter\n"
+     ]
+    }
+   ],
+   "source": [
+    "input_text = format_input(data[\"test\"][0])\n",
+    "\n",
+    "print(\"input (test):\", input_text)\n",
+    "\n",
+    "print(\"real answer:\", data[\"test\"][0][\"output\"])\n",
+    "\n",
+    "print(\"--------------------------\\n\")\n",
+    "\n",
+    "print(\"model's answer: \\n\")\n",
+    "print(inference(input_text, model, tokenizer))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "d1489173-84af-4c8e-b66b-0cdbe42c7ea7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_data = data[\"test\"].shuffle(seed=199).select(range(1000))\n",
+    "\n",
+    "data_list = []\n",
+    "\n",
+    "for entry in test_data:\n",
+    "    input_text = format_input(entry)\n",
+    "    #print(input_text)\n",
+    "    response_text = inference(input_text, model, tokenizer)\n",
+    "    #print(response_text)\n",
+    "    data = {\n",
+    "        \"instruction\":entry[\"instruction\"],\n",
+    "         \"input\":entry[\"input\"],\n",
+    "         \"output\":entry[\"output\"],\n",
+    "        \"model_response\":response_text\n",
+    "    }\n",
+    "\n",
+    "    data_list.append(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "39275fe6-ac3b-4558-9f4c-2853a41d48c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 定义输出文件路径\n",
+    "output_file = 'llama-sft-2.json'\n",
+    "\n",
+    "# 将 Dataset 对象导出为 JSON 文件\n",
+    "# test_data.to_json(output_file)\n",
+    "with open(output_file, \"w\") as file:\n",
+    "    json.dump(data_list, file, indent=4)  # \"indent\" for pretty-printing\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "7ffaba65-a270-4433-b234-932f5e288f7c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'▁prom oter'"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\" \".join(tokenizer.tokenize(\"promoter\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a7e373a4-6857-4874-b2da-58da2928925d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites |||||||||||| Courses\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites |||||||||||| Court\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites |||||||||||| Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites |||||||||||| Coursing\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Acceptor Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites |||||||||||| Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Donor Sites |||||||||||| D Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| C promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-Splice Sites |||||||||||| Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Acceptor Sites |||||||||||| Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites |||||||||||| Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Acceptor Sites |||||||||||| Splice Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  promoter\n",
+      "Non-Splice Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-Splice Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites ||||||||||||  Acceptor Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "promoter ||||||||||||  promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Donor Sites ||||||||||||  Acceptor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Non-Splice Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-Splice Sites ||||||||||||  Non-Splice Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter |||||||||||| promoter\n",
+      "Donor Sites |||||||||||| Acceptor Sites\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Acceptor Sites ||||||||||||  Donor Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Acceptor Sites |||||||||||| Donor Sites\n",
+      "promoter |||||||||||| promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Donor Sites ||||||||||||  Donor Sites\n",
+      "Binding Sites ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "promoter |||||||||||| promoter\n",
+      "Background Sequences ||||||||||||  Binding Sites\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "Background Sequences ||||||||||||  Background Sequences\n",
+      "promoter |||||||||||| promoter\n",
+      "Non-promoter ||||||||||||  Non-promoter\n",
+      "presicion 0.739 same 0.253\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "\n",
+    "\n",
+    "with open(output_file, \"r\") as file:\n",
+    "    test_data = json.load(file)\n",
+    "\n",
+    "all_num = len(test_data)\n",
+    "right_sum = 0\n",
+    "same_sum = 0\n",
+    "for item in test_data:\n",
+    "    output = item[\"output\"]\n",
+    "    #output = \" \".join(tokenizer.tokenize(output))\n",
+    "    model_response = item[\"model_response\"]\n",
+    "\n",
+    "    print(output,\"||||||||||||\", model_response)\n",
+    "\n",
+    "    if model_response == output: #same it\n",
+    "        same_sum = same_sum + 1\n",
+    "        \n",
+    "    if output.find(\"Non\")==-1: # no Non\n",
+    "        if model_response.find(output)!=-1 and model_response.find(\"Non\")==-1: #find it, but no Non\n",
+    "            right_sum = right_sum + 1\n",
+    "    else:\n",
+    "        if model_response.find(output)!=-1: #find it\n",
+    "            right_sum = right_sum + 1\n",
+    "\n",
+    "\n",
+    "print(\"presicion\", right_sum/all_num, \"same\", same_sum/all_num)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "294d46f3-2f5b-4e55-ae41-081d5195f5e2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

04-gene-sft/merge_llama_with_dna_lora.py

@@ -0,0 +1,367 @@
+"""
+Usage: 
+python merge_llama_with_chinese_lora.py \
+    --base_model path/to/llama/model \
+    --lora_model path/to/first/lora/model [path/to/second/lora/model] \
+    --output_type [pth|huggingface] \
+    --output_dir path/to/output/dir
+"""
+
+import os
+
+# 设置环境变量
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+# 打印环境变量以确认设置成功
+print(os.environ.get('HF_ENDPOINT'))
+
+import argparse
+import json
+import os
+import gc
+import torch
+import peft
+from peft import PeftModel
+from transformers import LlamaForCausalLM, LlamaTokenizer
+from huggingface_hub import hf_hub_download
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--base_model', default=None, required=True,
+                    type=str, help="Please specify a base_model")
+parser.add_argument('--lora_model', default=None, required=True,
+                    type=str, help="Please specify LoRA models to be merged (ordered); use commas to separate multiple LoRA models.")
+parser.add_argument('--offload_dir', default=None, type=str,
+                    help="(Optional) Please specify a temp folder for offloading (useful for low-RAM machines). Default None (disable offload).")
+parser.add_argument('--output_type', default='pth',choices=['pth','huggingface'], type=str,
+                    help="save the merged model in pth or huggingface format.")
+parser.add_argument('--output_dir', default='./', type=str)
+
+
+emb_to_model_size = {
+    4096 : '7B',
+    5120 : '13B',
+    6656 : '33B',
+    8192 : '65B',
+}
+num_shards_of_models = {'7B': 1, '13B': 2, '33B': 4, '65B': 8}
+params_of_models = {
+    '7B':
+        {
+        "dim": 4096,
+        "multiple_of": 256,
+        "n_heads": 32,
+        "n_layers": 32,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '13B':
+        {
+        "dim": 5120,
+        "multiple_of": 256,
+        "n_heads": 40,
+        "n_layers": 40,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '33B':
+        {
+        "dim": 6656,
+        "multiple_of": 256,
+        "n_heads": 52,
+        "n_layers": 60,
+        "norm_eps": 1e-06,
+        "vocab_size": -1,
+        },
+    '65B':
+        {
+        "dim": 8192,
+        "multiple_of": 256,
+        "n_heads": 64,
+        "n_layers": 80,
+        "norm_eps": 1e-05,
+        "vocab_size": -1,
+        },
+}
+
+def transpose(weight, fan_in_fan_out):
+    return weight.T if fan_in_fan_out else weight
+
+# Borrowed and modified from https://github.com/tloen/alpaca-lora
+def translate_state_dict_key(k):
+    k = k.replace("base_model.model.", "")
+    if k == "model.embed_tokens.weight":
+        return "tok_embeddings.weight"
+    elif k == "model.norm.weight":
+        return "norm.weight"
+    elif k == "lm_head.weight":
+        return "output.weight"
+    elif k.startswith("model.layers."):
+        layer = k.split(".")[2]
+        if k.endswith(".self_attn.q_proj.weight"):
+            return f"layers.{layer}.attention.wq.weight"
+        elif k.endswith(".self_attn.k_proj.weight"):
+            return f"layers.{layer}.attention.wk.weight"
+        elif k.endswith(".self_attn.v_proj.weight"):
+            return f"layers.{layer}.attention.wv.weight"
+        elif k.endswith(".self_attn.o_proj.weight"):
+            return f"layers.{layer}.attention.wo.weight"
+        elif k.endswith(".mlp.gate_proj.weight"):
+            return f"layers.{layer}.feed_forward.w1.weight"
+        elif k.endswith(".mlp.down_proj.weight"):
+            return f"layers.{layer}.feed_forward.w2.weight"
+        elif k.endswith(".mlp.up_proj.weight"):
+            return f"layers.{layer}.feed_forward.w3.weight"
+        elif k.endswith(".input_layernorm.weight"):
+            return f"layers.{layer}.attention_norm.weight"
+        elif k.endswith(".post_attention_layernorm.weight"):
+            return f"layers.{layer}.ffn_norm.weight"
+        elif k.endswith("rotary_emb.inv_freq") or "lora" in k:
+            return None
+        else:
+            print(layer, k)
+            raise NotImplementedError
+    else:
+        print(k)
+        raise NotImplementedError
+
+
+def unpermute(w):
+    return (
+        w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)
+    )
+
+
+def save_shards(model_sd, num_shards: int):
+    # Add the no_grad context manager
+    with torch.no_grad():
+        if num_shards == 1:
+            new_state_dict = {}
+            for k, v in model_sd.items():
+                new_k = translate_state_dict_key(k)
+                if new_k is not None:
+                    if "wq" in new_k or "wk" in new_k:
+                        new_state_dict[new_k] = unpermute(v)
+                    else:
+                        new_state_dict[new_k] = v
+
+            os.makedirs(output_dir, exist_ok=True)
+            print(f"Saving shard 1 of {num_shards} into {output_dir}/consolidated.00.pth")
+            torch.save(new_state_dict, output_dir + "/consolidated.00.pth")
+            with open(output_dir + "/params.json", "w") as f:
+                json.dump(params, f)
+        else:
+            new_state_dicts = [dict() for _ in range(num_shards)]
+            for k in list(model_sd.keys()):
+                v = model_sd[k]
+                new_k = translate_state_dict_key(k)
+                if new_k is not None:
+                    if new_k=='tok_embeddings.weight':
+                        print(f"Processing {new_k}")
+                        assert v.size(1)%num_shards==0
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+                    elif new_k=='output.weight':
+                        print(f"Processing {new_k}")
+                        if v.size(0)%num_shards==0:
+                            splits = v.split(v.size(0)//num_shards,dim=0)
+                        else:
+                            size_list = [v.size(0)//num_shards] * num_shards
+                            size_list[-1] += v.size(0)%num_shards
+                            splits = v.split(size_list, dim=0) # 13B: size_list == [24976,24977]
+                    elif new_k=='norm.weight':
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+                    elif 'ffn_norm.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+                    elif 'attention_norm.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = [v] * num_shards
+
+
+                    elif 'w1.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+                    elif 'w2.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+                    elif 'w3.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+
+
+                    elif 'wo.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(1)//num_shards,dim=1)
+
+                    elif 'wv.weight' in new_k:
+                        print(f"Processing {new_k}")
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+
+                    elif "wq.weight" in new_k or "wk.weight" in new_k:
+                        print(f"Processing {new_k}")
+                        v = unpermute(v)
+                        splits = v.split(v.size(0)//num_shards,dim=0)
+                    else:
+                        print(f"Unexpected key {new_k}")
+                        raise ValueError
+                    for sd,split in zip(new_state_dicts,splits):
+                        sd[new_k] = split.clone()
+                        del split
+                    del splits
+                del model_sd[k],v
+                gc.collect()    # Effectively enforce garbage collection
+
+            os.makedirs(output_dir, exist_ok=True)
+            for i,new_state_dict in enumerate(new_state_dicts):
+                print(f"Saving shard {i+1} of {num_shards} into {output_dir}/consolidated.0{i}.pth")
+                torch.save(new_state_dict, output_dir + f"/consolidated.0{i}.pth")
+            with open(output_dir + "/params.json", "w") as f:
+                print(f"Saving params.json into {output_dir}/params.json")
+                json.dump(params, f)
+
+
+if __name__=='__main__':
+
+    args = parser.parse_args()
+    base_model_path = args.base_model
+    lora_model_paths = [s.strip() for s in args.lora_model.split(',') if len(s.strip())!=0]
+    output_dir = args.output_dir
+    output_type = args.output_type
+    offload_dir = args.offload_dir
+
+    print(f"Base model: {base_model_path}")
+    print(f"LoRA model(s) {lora_model_paths}:")
+
+    if offload_dir is not None:
+        # Load with offloading, which is useful for low-RAM machines.
+        # Note that if you have enough RAM, please use original method instead, as it is faster.
+        base_model = LlamaForCausalLM.from_pretrained(
+            base_model_path,
+            load_in_8bit=False,
+            torch_dtype=torch.float16,
+            offload_folder=offload_dir,
+            offload_state_dict=True,
+            low_cpu_mem_usage=True,
+            device_map={"": "cpu"},
+        )
+    else:
+        # Original method without offloading
+        base_model = LlamaForCausalLM.from_pretrained(
+            base_model_path,
+            load_in_8bit=False,
+            torch_dtype=torch.float16,
+            device_map={"": "cpu"},
+            cache_dir=None,  # 不使用缓存目录
+            force_download=False,  # 禁止从远程下载
+            local_files_only=True  # 强制仅从本地文件加载                                    
+        )
+
+    ## infer the model size from the checkpoint
+    embedding_size = base_model.get_input_embeddings().weight.size(1)
+    model_size = emb_to_model_size[embedding_size]
+    print(f"Peft version: {peft.__version__}")
+    print(f"Loading LoRA for {model_size} model")
+
+    lora_model = None
+    lora_model_sd = None
+    for lora_index, lora_model_path in enumerate(lora_model_paths):
+        print(f"Loading LoRA {lora_model_path}...")
+        tokenizer = LlamaTokenizer.from_pretrained(lora_model_path, 
+                                                   cache_dir=None,  # 不使用缓存目录
+                                                   force_download=False,  # 禁止从远程下载
+                                                   local_files_only=True  # 强制仅从本地文件加载
+                                                  )
+        
+        print(f"base_model vocab size: {base_model.get_input_embeddings().weight.size(0)}")
+        print(f"tokenizer vocab size: {len(tokenizer)}")
+
+        model_vocab_size = base_model.get_input_embeddings().weight.size(0)
+        assert len(tokenizer) >= model_vocab_size, \
+        (f"The vocab size of the tokenizer {len(tokenizer)} is smaller than the vocab size of the base model {model_vocab_size}\n"
+        "This is not the intended use. Please check your model and tokenizer.")
+        if model_vocab_size != len(tokenizer):
+            base_model.resize_token_embeddings(len(tokenizer))
+            print(f"Extended vocabulary size to {len(tokenizer)}")
+
+        first_weight = base_model.model.layers[0].self_attn.q_proj.weight
+        first_weight_old = first_weight.clone()
+
+        print(f"Loading LoRA weights")
+        if hasattr(peft.LoraModel,'merge_and_unload'):
+            try:
+                lora_model = PeftModel.from_pretrained(
+                    base_model,
+                    lora_model_path,
+                    device_map={"": "cpu"},
+                    torch_dtype=torch.float16,
+                    local_files_only=True
+                )
+            except RuntimeError as e:
+                if '[49953, 4096]' in str(e):
+                    print("The vocab size of the tokenizer does not match the vocab size of the LoRA weight. \n"
+                           "Did you misuse the LLaMA tokenizer with the Alpaca-LoRA weight?\n"
+                           "Make sure that you use LLaMA tokenizer with the LLaMA-LoRA weight and Alpaca tokenizer with the Alpaca-LoRA weight!")
+                raise e
+            assert torch.allclose(first_weight_old, first_weight)
+            print(f"Merging with merge_and_unload...")
+            base_model = lora_model.merge_and_unload()
+        else:
+            base_model_sd = base_model.state_dict()
+            try:
+                lora_model_sd = torch.load(os.path.join(lora_model_path,'adapter_model.bin'),map_location='cpu')
+            except FileNotFoundError:
+                print("Cannot find lora model on the disk. Downloading lora model from hub...")
+                filename = hf_hub_download(repo_id=lora_model_path,filename='adapter_model.bin')
+                lora_model_sd = torch.load(filename,map_location='cpu')
+            if 'base_model.model.model.embed_tokens.weight' in lora_model_sd:
+                assert lora_model_sd['base_model.model.model.embed_tokens.weight'].shape[0]==len(tokenizer), \
+                ("The vocab size of the tokenizer does not match the vocab size of the LoRA weight. \n"
+                "Did you misuse the LLaMA tokenizer with the Alpaca-LoRA weight?\n"
+                "Make sure that you use LLaMA tokenizer with the LLaMA-LoRA weight and Alpaca tokenizer with the Alpaca-LoRA weight!")
+
+            lora_config = peft.LoraConfig.from_pretrained(lora_model_path)
+            lora_scaling = lora_config.lora_alpha / lora_config.r
+            fan_in_fan_out = lora_config.fan_in_fan_out
+            lora_keys = [k for k in lora_model_sd if 'lora_A' in k]
+            non_lora_keys = [k for k in lora_model_sd if not 'lora_' in k]
+
+            for k in non_lora_keys:
+                print(f"merging {k}")
+                original_k = k.replace('base_model.model.','')
+                base_model_sd[original_k].copy_(lora_model_sd[k])
+
+            for k in lora_keys:
+                print(f"merging {k}")
+                original_key = k.replace('.lora_A','').replace('base_model.model.','')
+                assert original_key in base_model_sd
+                lora_a_key = k
+                lora_b_key = k.replace('lora_A','lora_B')
+                base_model_sd[original_key] += (
+                    transpose(lora_model_sd[lora_b_key].float() @ lora_model_sd[lora_a_key].float(),fan_in_fan_out) * lora_scaling
+                )
+                assert base_model_sd[original_key].dtype == torch.float16
+
+        # did we do anything?
+        assert not torch.allclose(first_weight_old, first_weight)
+
+    tokenizer.save_pretrained(output_dir)
+
+    if output_type=='huggingface':
+        print("Saving to Hugging Face format...")
+        LlamaForCausalLM.save_pretrained(base_model, output_dir) #, state_dict=deloreanized_sd)
+    else: # output_type=='pth
+        print("Saving to pth format...")
+
+        base_model_sd = base_model.state_dict()
+        del lora_model, base_model, lora_model_sd
+
+        params = params_of_models[model_size]
+        num_shards = num_shards_of_models[model_size]
+        n_layers = params["n_layers"]
+        n_heads = params["n_heads"]
+        dim = params["dim"]
+        dims_per_head = dim // n_heads
+        base = 10000.0
+        inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))
+
+        save_shards(model_sd=base_model_sd, num_shards=num_shards)

04-gene-sft/merge_pt_model.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+python merge_llama_with_dna_lora.py \
+    --base_model llama-7b-hf \
+    --lora_model dnahlm_llama_7b/pt_lora_model \
+    --output_type huggingface \
+    --output_dir dnahlm-merge-hf

04-gene-sft/merge_sft_model.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+python merge_llama_with_dna_lora.py \
+    --base_model dnahlm-merge-hf \
+    --lora_model dnahlm-llama7b-sft/sft_lora_model \
+    --output_type huggingface \
+    --output_dir dnahlm-llama-7b-sft-v0

04-gene-sft/merged_gene_eng_tokenizer_hf/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

04-gene-sft/merged_gene_eng_tokenizer_hf/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f9bfd4fff4bf6132695295a6443cf0c9fdf923ba58ea628e5efbeb25ce95aed
+size 1360570

04-gene-sft/merged_gene_eng_tokenizer_hf/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

04-gene-sft/merged_gene_eng_tokenizer_sp/gene_eng_llama_tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f9bfd4fff4bf6132695295a6443cf0c9fdf923ba58ea628e5efbeb25ce95aed
+size 1360570

04-gene-sft/run_clm_pt_with_peft.py

@@ -68,7 +68,11 @@ class SavePeftModelCallback(transformers.TrainerCallback):
 
         peft_model_path = os.path.join(checkpoint_folder, "pt_lora_model")
         kwargs["model"].save_pretrained(peft_model_path)
-        kwargs["tokenizer"].save_pretrained(peft_model_path)
+
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
 
     def on_save(self, args, state, control, **kwargs):
         self.save_model(args, state, kwargs)
@@ -77,7 +81,11 @@ class SavePeftModelCallback(transformers.TrainerCallback):
     def on_train_end(self, args, state, control, **kwargs):
         peft_model_path = os.path.join(args.output_dir, "pt_lora_model")
         kwargs["model"].save_pretrained(peft_model_path)
-        kwargs["tokenizer"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
 
 
 def accuracy(predictions, references, normalize=True, sample_weight=None):

04-gene-sft/run_clm_sft_with_peft.py

@@ -69,7 +69,12 @@ class SavePeftModelCallback(transformers.TrainerCallback):
 
         peft_model_path = os.path.join(checkpoint_folder, "sft_lora_model")
         kwargs["model"].save_pretrained(peft_model_path)
-        kwargs["tokenizer"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
 
     def on_save(self, args, state, control, **kwargs):
         self.save_model(args, state, kwargs)
@@ -78,7 +83,12 @@ class SavePeftModelCallback(transformers.TrainerCallback):
     def on_train_end(self, args, state, control, **kwargs):
         peft_model_path = os.path.join(args.output_dir, "sft_lora_model")
         kwargs["model"].save_pretrained(peft_model_path)
-        kwargs["tokenizer"].save_pretrained(peft_model_path)
+        
+        if "tokenizer" in kwargs:
+            kwargs["tokenizer"].save_pretrained(peft_model_path)
+        else:
+            kwargs["processing_class"].save_pretrained(peft_model_path)
+
 
 
 @dataclass

04-gene-sft/run_sft.sh

@@ -56,5 +56,4 @@ torchrun --nnodes 1 --nproc_per_node 6 run_clm_sft_with_peft.py \
     --torch_dtype float16 \
     --validation_file ${validation_file} \
     --gradient_checkpointing \
-    --ddp_find_unused_parameters False \
-    --resume_from_checkpoint dnahlm-llama7b-sft/checkpoint-464
+    --ddp_find_unused_parameters False 

04-gene-sft/train_data/dna_1g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d950f86ccdb368f4652795117d23898dbccfce5a18a0ee84f78aebc43e8742
+size 1080669660

04-gene-sft/train_data/english_500m.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085ebb9d461cae266410953bcd2d07d9a08d50cd93db24d5c3e15d38275cd8cd
+size 541727453

04-gene-sft/train_data/protein_1g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1c361441538520a5501605fa483970b80d72b5dbb28dbe5276890c8632ba1d4
+size 1059750637