Model save

README.md

@@ -0,0 +1,67 @@
+---
+library_name: peft
+license: apache-2.0
+base_model: HuggingFaceTB/SmolLM2-1.7B-Instruct
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: smollm2-1.7b-instr-sft-cai
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# smollm2-1.7b-instr-sft-cai
+
+This model is a fine-tuned version of [HuggingFaceTB/SmolLM2-1.7B-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2565
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.001
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 64
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.2481        | 0.9913 | 100  | 1.2565          |
+
+
+### Framework versions
+
+- PEFT 0.14.0
+- Transformers 4.47.1
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9913258983890955,
+    "total_flos": 1.280941246906368e+17,
+    "train_loss": 1.281534285545349,
+    "train_runtime": 8293.439,
+    "train_samples": 11456,
+    "train_samples_per_second": 0.778,
+    "train_steps_per_second": 0.012
+}

runs/Jan03_10-24-26_pkaiju/events.out.tfevents.1735899878.pkaiju.20743.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:452386a603f212b01bda9f08cf6032d49ce27a045c25a3d079de87bf511b6926
-size 11195
+oid sha256:29ab162bfb323ca537070d83bfff3b154324db9ba70381fd48eacc1a74163b9b
+size 11809

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9913258983890955,
+    "total_flos": 1.280941246906368e+17,
+    "train_loss": 1.281534285545349,
+    "train_runtime": 8293.439,
+    "train_samples": 11456,
+    "train_samples_per_second": 0.778,
+    "train_steps_per_second": 0.012
+}

trainer_state.json

@@ -0,0 +1,197 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9913258983890955,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009913258983890954,
+      "grad_norm": 0.06124192103743553,
+      "learning_rate": 0.0001,
+      "loss": 1.419,
+      "step": 1
+    },
+    {
+      "epoch": 0.04956629491945477,
+      "grad_norm": 0.059546224772930145,
+      "learning_rate": 0.0005,
+      "loss": 1.4329,
+      "step": 5
+    },
+    {
+      "epoch": 0.09913258983890955,
+      "grad_norm": 0.03903435170650482,
+      "learning_rate": 0.001,
+      "loss": 1.3851,
+      "step": 10
+    },
+    {
+      "epoch": 0.14869888475836432,
+      "grad_norm": 0.07022791355848312,
+      "learning_rate": 0.000992403876506104,
+      "loss": 1.3227,
+      "step": 15
+    },
+    {
+      "epoch": 0.1982651796778191,
+      "grad_norm": 0.039068832993507385,
+      "learning_rate": 0.0009698463103929542,
+      "loss": 1.3103,
+      "step": 20
+    },
+    {
+      "epoch": 0.24783147459727387,
+      "grad_norm": 0.02727026864886284,
+      "learning_rate": 0.0009330127018922195,
+      "loss": 1.3104,
+      "step": 25
+    },
+    {
+      "epoch": 0.29739776951672864,
+      "grad_norm": 0.03437395021319389,
+      "learning_rate": 0.000883022221559489,
+      "loss": 1.2928,
+      "step": 30
+    },
+    {
+      "epoch": 0.3469640644361834,
+      "grad_norm": 0.02124294452369213,
+      "learning_rate": 0.0008213938048432696,
+      "loss": 1.2983,
+      "step": 35
+    },
+    {
+      "epoch": 0.3965303593556382,
+      "grad_norm": 0.025805862620472908,
+      "learning_rate": 0.00075,
+      "loss": 1.2597,
+      "step": 40
+    },
+    {
+      "epoch": 0.44609665427509293,
+      "grad_norm": 0.02471889927983284,
+      "learning_rate": 0.0006710100716628344,
+      "loss": 1.2451,
+      "step": 45
+    },
+    {
+      "epoch": 0.49566294919454773,
+      "grad_norm": 0.024004101753234863,
+      "learning_rate": 0.0005868240888334653,
+      "loss": 1.2512,
+      "step": 50
+    },
+    {
+      "epoch": 0.5452292441140025,
+      "grad_norm": 0.019842401146888733,
+      "learning_rate": 0.0005,
+      "loss": 1.2609,
+      "step": 55
+    },
+    {
+      "epoch": 0.5947955390334573,
+      "grad_norm": 0.024189768359065056,
+      "learning_rate": 0.00041317591116653486,
+      "loss": 1.2482,
+      "step": 60
+    },
+    {
+      "epoch": 0.644361833952912,
+      "grad_norm": 0.03272519260644913,
+      "learning_rate": 0.0003289899283371657,
+      "loss": 1.2624,
+      "step": 65
+    },
+    {
+      "epoch": 0.6939281288723668,
+      "grad_norm": 0.026437299326062202,
+      "learning_rate": 0.0002500000000000001,
+      "loss": 1.2476,
+      "step": 70
+    },
+    {
+      "epoch": 0.7434944237918215,
+      "grad_norm": 0.01857222244143486,
+      "learning_rate": 0.0001786061951567303,
+      "loss": 1.2496,
+      "step": 75
+    },
+    {
+      "epoch": 0.7930607187112764,
+      "grad_norm": 0.02758738212287426,
+      "learning_rate": 0.00011697777844051105,
+      "loss": 1.2689,
+      "step": 80
+    },
+    {
+      "epoch": 0.8426270136307311,
+      "grad_norm": 0.022616183385252953,
+      "learning_rate": 6.698729810778065e-05,
+      "loss": 1.2479,
+      "step": 85
+    },
+    {
+      "epoch": 0.8921933085501859,
+      "grad_norm": 0.028509799391031265,
+      "learning_rate": 3.0153689607045842e-05,
+      "loss": 1.2496,
+      "step": 90
+    },
+    {
+      "epoch": 0.9417596034696406,
+      "grad_norm": 0.02118438109755516,
+      "learning_rate": 7.59612349389599e-06,
+      "loss": 1.2417,
+      "step": 95
+    },
+    {
+      "epoch": 0.9913258983890955,
+      "grad_norm": 0.017582964152097702,
+      "learning_rate": 0.0,
+      "loss": 1.2481,
+      "step": 100
+    },
+    {
+      "epoch": 0.9913258983890955,
+      "eval_loss": 1.2565181255340576,
+      "eval_runtime": 3369.8446,
+      "eval_samples_per_second": 4.224,
+      "eval_steps_per_second": 0.528,
+      "step": 100
+    },
+    {
+      "epoch": 0.9913258983890955,
+      "step": 100,
+      "total_flos": 1.280941246906368e+17,
+      "train_loss": 1.281534285545349,
+      "train_runtime": 8293.439,
+      "train_samples_per_second": 0.778,
+      "train_steps_per_second": 0.012
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.280941246906368e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}