Upload folder using huggingface_hub

config.json

@@ -21,5 +21,5 @@
   "pad_token_id": 0,
   "relative_attention_num_buckets": 32,
   "tie_word_embeddings": false,
-  "vocab_size": 32100
+  "vocab_size": 64100
 }

hydra/config.yaml

@@ -0,0 +1,47 @@
+mode: pt
+device: gpu
+precision: bf16
+eval_only: false
+predict_only: false
+seed: 2137
+model:
+  klass: hf_t5
+  name: /home/jovyan/bert-train/nanot5/base_slovak_model/
+  overwrite:
+    dropout_rate: 0.0
+  add_config:
+    is_bf16: false
+  checkpoint_path: ''
+  random_init: true
+  compile: false
+data:
+  input_length: 512
+  mlm_probability: 0.15
+  mean_noise_span_length: 3.0
+  num_workers: 2
+optim:
+  name: adamwscale
+  base_lr: 0.02
+  batch_size: 128
+  total_steps: 65536
+  epochs: -1
+  warmup_steps: 10000
+  lr_scheduler: cosine
+  weight_decay: 0.0
+  grad_clip: 1.0
+  grad_acc: 4
+  final_cosine: 1.0e-05
+eval:
+  every_steps: 5000
+  steps: 500
+checkpoint:
+  every_steps: 20000
+logging:
+  neptune: false
+  neptune_creds:
+    project: null
+    api_token: null
+    tags: ''
+  every_steps: 100
+  grad_l2: true
+  weights_l2: true

hydra/hydra.yaml

@@ -0,0 +1,163 @@
+hydra:
+  run:
+    dir: ./logs/${now:%Y-%m-%d}/${now:%H-%M-%S}-${logging.neptune_creds.tags}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - optim.name=adamwscale
+    - optim.lr_scheduler=cosine
+    - model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/
+    - optim.grad_acc=4
+    - model.klass=hf_t5
+    - eval.every_steps=5000
+    - checkpoint.every_steps=20000
+  job:
+    name: main
+    chdir: true
+    override_dirname: checkpoint.every_steps=20000,eval.every_steps=5000,model.klass=hf_t5,model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/,optim.grad_acc=4,optim.lr_scheduler=cosine,optim.name=adamwscale
+    id: ???
+    num: ???
+    config_name: default
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/jovyan/nanoT5
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/jovyan/nanoT5/nanoT5/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/jovyan/nanoT5/logs/2024-01-02/07-29-30-
+    choices:
+      local_env: default
+      task: pt
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

hydra/overrides.yaml

@@ -0,0 +1,7 @@
+- optim.name=adamwscale
+- optim.lr_scheduler=cosine
+- model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/
+- optim.grad_acc=4
+- model.klass=hf_t5
+- eval.every_steps=5000
+- checkpoint.every_steps=20000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac4258d6a76c6e7a0687bc7af68ce34538f3b67e4387678e96eb6a2d29114a9e
-size 990173032
+oid sha256:8adcaa3befd91b9080f22d00466debbce77b0989eb3b038269f83aad1ee5e934
+size 1186781032

optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d9f3304271a05b8894b45869b7bae03ae655aa7615410d7ee722ab02064ceea
+size 2373662661

random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a429eefea0f594d70075b0d57721b18d7e6e4d12263218ee30780a6e613c04cd
+size 14663

scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b51bb232eab4bf0dd13cf507f602d257c3596e6828466eb03664de335c46223
+size 819

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89ffef2172838c61ae45e7f6174233eec5c315c22646537d077c4e6b5dc76cd9
-size 812913
+oid sha256:353edf8f042581ee554bb883dd4a19e0888d778553e3fd5ca9c97bb76434406b
+size 1408434