doc updates

README.md

@@ -66,54 +66,60 @@ The training hyperparameters are those suggested by Adelani et al. (2022) in the
 release](https://github.com/masakhane-io/lafand-mt), which gave their best results for machine
 translation of several African languages.
 
-More specifically, we use the [example training
-script](https://github.com/huggingface/transformers/blob/06886d5a684228a695b29645993b3be55190bd9c/examples/pytorch/translation/run_translation.py)
-provided by 🤗 Transformers for fine-tuning mBART with the following command
+More specifically, we train this model with [zeldarose](https://github.com/LoicGrobol/zeldarose) with the following parameters
 
 ```bash
-python run_translation.py \
-  --model_name_or_path facebook/m2m100_418M  \
-  --do_train \
-  --train_file {path_to_training_data} \
-  --source_lang br \
-  --target_lang fr \
-  --output_dir {path_to_model}\
-  --per_device_train_batch_size=8 \
-  --overwrite_output_dir \
-  --forced_bos_token fr \
-  --save_steps 4096 \
-  --fp16 \
-  --num_train_epochs 4
+zeldarose transformer \
+   --config train_config.toml \
+   --tokenizer "facebook/m2m100_418M" --pretrained-model "facebook/m2m100_418M" \
+   --out-dir m2m100_418M+br-fr --model-name m2m100_418M+br-fr \
+   --strategy ddp --accelerator gpu --num-devices 4 --device-batch-size 2 --num-workers 8\
+   --max-epochs 16 --precision 16 --tf32-mode medium \
+   --val-data {val_path}.jsonl \
+   {train_path}.jsonl
+
 ```
 
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
 
-- `learning_rate`: 5e-05
-- `train_batch_size`: 8
-- `eval_batch_size`: 8
-- `seed`: 42
-- `optimizer`: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- `lr_scheduler_type`: linear
-- `num_epochs`: 4.0
+```toml
+[task]
+change_ratio = 0.3
+denoise_langs = []
+poisson_lambda = 3.0
+source_langs = ["br"]
+target_langs = ["fr"]
+
+[tuning]
+batch_size = 16
+betas = [0.9, 0.999]
+epsilon = 1e-8
+learning_rate = 5e-5
+gradient_clipping = 1.0
+lr_decay_steps = -1
+warmup_steps = 1024
+```
 
 ### Framework versions
 
-- Transformers 4.24.0
-- Pytorch 1.13.0
-- Datasets 2.6.1
-- Tokenizers 0.13.1
+- Transformers 4.26.1
+- Pytorch 1.12.1
+- Datasets 2.10.0
+- Tokenizers 0.13.2
+- Pytorch-lightning 1.9.3
+- Zeldarose [c6456ead](https://github.com/LoicGrobol/spertiniite/commit/c6456ead3649c4e6ddfb4a5a74b40f344eded09f)
 
 ### Carbon emissions
 
 At this time, we estimate emissions of a rough 300 gCO<sub>2</sub> per fine-tuning run. So far, we
 account for
 
-- Fine-tuning the 2 released versions
-- 5 development runs
+- Fine-tuning the 3 released versions
+- 8 development runs
 
-So far, the equivalent carbon emissions for this model are approximately 2100 gCO<sub>2</sub>.
+So far, the equivalent carbon emissions for this model are approximately 3300 gCO<sub>2</sub>.
 
 ## References
 

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 128112
+}

all_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 4.0,
-    "train_loss": 1.4005291703168083,
-    "train_runtime": 11994.4751,
-    "train_samples": 54393,
-    "train_samples_per_second": 18.139,
-    "train_steps_per_second": 1.134
-}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "early_stopping": true,
+  "eos_token_id": 2,
+  "max_length": 200,
+  "num_beams": 5,
+  "pad_token_id": 1,
+  "transformers_version": "4.26.1"
+}

train_config.toml

@@ -0,0 +1,19 @@
+type = "mbart"
+
+[task]
+change_ratio = 0.3
+denoise_langs = []
+poisson_lambda = 3.0
+source_langs = ["br"]
+target_langs = ["fr"]
+
+[tuning]
+batch_size = 16
+betas = [0.9, 0.999]
+epsilon = 1e-8
+learning_rate = 5e-5
+gradient_clipping = 1.0
+# Uncomment these for a more complex training setup
+lr_decay_steps = -1
+warmup_steps = 1024
+# weight_decay = 1e-5

train_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 4.0,
-    "train_loss": 1.4005291703168083,
-    "train_runtime": 11994.4751,
-    "train_samples": 54393,
-    "train_samples_per_second": 18.139,
-    "train_steps_per_second": 1.134
-}

trainer_state.json

@@ -1,187 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 4.0,
-  "global_step": 13600,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.15,
-      "learning_rate": 4.816176470588236e-05,
-      "loss": 2.6313,
-      "step": 500
-    },
-    {
-      "epoch": 0.29,
-      "learning_rate": 4.632352941176471e-05,
-      "loss": 2.2069,
-      "step": 1000
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 4.448529411764706e-05,
-      "loss": 2.035,
-      "step": 1500
-    },
-    {
-      "epoch": 0.59,
-      "learning_rate": 4.2647058823529415e-05,
-      "loss": 1.9491,
-      "step": 2000
-    },
-    {
-      "epoch": 0.74,
-      "learning_rate": 4.08125e-05,
-      "loss": 1.8742,
-      "step": 2500
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 3.897426470588236e-05,
-      "loss": 1.8387,
-      "step": 3000
-    },
-    {
-      "epoch": 1.03,
-      "learning_rate": 3.713602941176471e-05,
-      "loss": 1.6941,
-      "step": 3500
-    },
-    {
-      "epoch": 1.18,
-      "learning_rate": 3.529779411764706e-05,
-      "loss": 1.5224,
-      "step": 4000
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 3.3459558823529415e-05,
-      "loss": 1.4897,
-      "step": 4500
-    },
-    {
-      "epoch": 1.47,
-      "learning_rate": 3.1621323529411765e-05,
-      "loss": 1.4445,
-      "step": 5000
-    },
-    {
-      "epoch": 1.62,
-      "learning_rate": 2.978308823529412e-05,
-      "loss": 1.4593,
-      "step": 5500
-    },
-    {
-      "epoch": 1.76,
-      "learning_rate": 2.7944852941176468e-05,
-      "loss": 1.4251,
-      "step": 6000
-    },
-    {
-      "epoch": 1.91,
-      "learning_rate": 2.6113970588235297e-05,
-      "loss": 1.39,
-      "step": 6500
-    },
-    {
-      "epoch": 2.06,
-      "learning_rate": 2.427573529411765e-05,
-      "loss": 1.2959,
-      "step": 7000
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 2.24375e-05,
-      "loss": 1.1621,
-      "step": 7500
-    },
-    {
-      "epoch": 2.35,
-      "learning_rate": 2.0599264705882353e-05,
-      "loss": 1.1374,
-      "step": 8000
-    },
-    {
-      "epoch": 2.5,
-      "learning_rate": 1.876102941176471e-05,
-      "loss": 1.1649,
-      "step": 8500
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 1.6926470588235294e-05,
-      "loss": 1.1513,
-      "step": 9000
-    },
-    {
-      "epoch": 2.79,
-      "learning_rate": 1.5088235294117647e-05,
-      "loss": 1.1463,
-      "step": 9500
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.3250000000000002e-05,
-      "loss": 1.1466,
-      "step": 10000
-    },
-    {
-      "epoch": 3.09,
-      "learning_rate": 1.1411764705882353e-05,
-      "loss": 1.0411,
-      "step": 10500
-    },
-    {
-      "epoch": 3.24,
-      "learning_rate": 9.573529411764706e-06,
-      "loss": 0.9581,
-      "step": 11000
-    },
-    {
-      "epoch": 3.38,
-      "learning_rate": 7.735294117647058e-06,
-      "loss": 0.9514,
-      "step": 11500
-    },
-    {
-      "epoch": 3.53,
-      "learning_rate": 5.897058823529412e-06,
-      "loss": 0.9429,
-      "step": 12000
-    },
-    {
-      "epoch": 3.68,
-      "learning_rate": 4.058823529411765e-06,
-      "loss": 0.9676,
-      "step": 12500
-    },
-    {
-      "epoch": 3.82,
-      "learning_rate": 2.2205882352941175e-06,
-      "loss": 0.9324,
-      "step": 13000
-    },
-    {
-      "epoch": 3.97,
-      "learning_rate": 3.8235294117647064e-07,
-      "loss": 0.9555,
-      "step": 13500
-    },
-    {
-      "epoch": 4.0,
-      "step": 13600,
-      "total_flos": 3.918346910230118e+16,
-      "train_loss": 1.4005291703168083,
-      "train_runtime": 11994.4751,
-      "train_samples_per_second": 18.139,
-      "train_steps_per_second": 1.134
-    }
-  ],
-  "max_steps": 13600,
-  "num_train_epochs": 4,
-  "total_flos": 3.918346910230118e+16,
-  "trial_name": null,
-  "trial_params": null
-}

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d7e19c4b52c1665d4e24c8332861794cb0354d00704d62e085c5f3112b7d82d7
-size 3579