clean

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/bench.slurm

@@ -1,111 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=bench_cluster
-#SBATCH --time=02:00:00
-#SBATCH --partition=hopper-prod
-#SBATCH --nodes=8
-#SBATCH --gres=gpu:8
-#SBATCH --qos=prod
-#SBATCH --ntasks-per-node=1
-#SBATCH --cpus-per-task=96
-#SBATCH --exclusive
-#SBATCH --output=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log.out
-#SBATCH --error=/fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log.out
-
-# Function to update status based on squeue output
-update_status() {
-    job_id=$1
-    status_file=$2
-    # For unknown reasons, it doenst update status for pending. It only works for running 
-    while true; do
-        job_status=$(squeue --job $job_id --noheader --format=%T)
-        echo "Job status: $job_status"
-        if [ -z "$job_status" ]; then
-            # Job has finished or is not found
-            break
-        elif [ "$job_status" = "RUNNING" ]; then
-            printf "running" > $status_file
-            break
-        fi
-        sleep 10
-    done
-}
-
-# Misc initializations.
-echo "========================"
-echo "START TIME: $(date)"
-source /fsx/ferdinandmom/miniforge3/etc/profile.d/conda.sh
-conda activate /fsx/ferdinandmom/miniforge3/envs/env-bench-cluster
-echo python3 version = $(python3 --version)
-echo "========================"
-
-# Slurm stuff
-export HOSTNAMES=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
-export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
-export MASTER_PORT=$((1024 + RANDOM % 64511))
-
-export TMPDIR=/scratch
-export HF_DATASETS_CACHE="/admin/home/ferdinand_mom/.cache"
-export CUBLAS_WORKSPACE_CONFIG=":4096:8"
-export CUDA_DEVICE_MAX_CONNECTIONS="1"
-
-huggingface-cli login --token $HUGGINGFACE_TOKEN
-
-
-NANOTRON_REPO="/fsx/ferdinandmom/ferdinand-hf/bench_cluster/nanotron"
-CMD="$NANOTRON_REPO/run_train.py --config-file /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/config.yaml"
-
-LAUNCHER="torchrun \
-   --nproc_per_node 8 \
-   --nnodes 8 \
-   --rdzv_endpoint ${MASTER_ADDR}:${MASTER_PORT} \
-   --rdzv_backend c10d \
-   --max_restarts 0 \
-   --tee 3 \
-   --node_rank ${SLURM_PROCID}"
-
-# Checkout the bench_cluster branch
-cd $NANOTRON_REPO
-git checkout bench_cluster
-cd ..
-# Get the current job ID
-job_id=${SLURM_JOB_ID}
-
-# Update status to "pending" or "running" in the background
-update_status $job_id /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt &
-
-# Run the main command
-srun -u $LAUNCHER $CMD
-exit_status=$?
-
-# Update status based on the exit status of `srun`
-if [ $exit_status -eq 0 ]; then
-    printf "completed" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt
-else
-    if grep -q "OutOfMemoryError" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt
-    elif grep -q " CUDA error: an illegal memory access" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log.out; then
-        printf "oom" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt
-    elif grep -q "Timeout at NCCL" /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log.out; then
-        printf "timeout" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt
-    else
-        printf "fail" > /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt
-    fi 
-fi
-
-# Run the report script if the job completed successfully
-if [ $exit_status -eq 0 ]; then
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16 --is_logs
-    python /fsx/ferdinandmom/ferdinand-hf/bench_cluster/main.py report --inp_dir /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16 --is_profiler
-fi
-
-
-# Push to hub the folder using huggingface_cli
-huggingface-cli upload nanotron/bench_cluster /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16 llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16 --commit-message "Upload llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16"
-
-# Verify the upload
-if [ $? -eq 0 ]; then
-    echo "Uploading to Huggingface Hub successful"
-else
-    echo "Failed to upload to Huggingface Hub"
-fi

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/config.yaml

@@ -1,90 +0,0 @@
-general:
-  project: bench_cluster
-  seed: 42
-model:
-  ddp_bucket_cap_mb: 25
-  dtype: bfloat16
-  init_method:
-    std: 0.025
-  make_vocab_size_divisible_by: 1
-  model_config:
-    bos_token_id: 1
-    eos_token_id: 2
-    hidden_act: silu
-    hidden_size: 2048
-    initializer_range: 0.02
-    intermediate_size: 4096
-    is_llama_config: true
-    max_position_embeddings: 4096
-    num_attention_heads: 32
-    num_hidden_layers: 24
-    num_key_value_heads: 32
-    pad_token_id: null
-    pretraining_tp: 1
-    rms_norm_eps: 1.0e-05
-    rope_scaling: null
-    rope_theta: 10000.0
-    tie_word_embeddings: true
-    use_cache: true
-    vocab_size: 50257
-optimizer:
-  accumulate_grad_in_fp32: true
-  clip_grad: 1.0
-  learning_rate_scheduler:
-    learning_rate: 0.0001
-    lr_decay_style: linear
-    lr_warmup_style: linear
-    lr_warmup_steps: 1
-    min_decay_lr: 1.0e-05
-  optimizer_factory:
-    adam_beta1: 0.9
-    adam_beta2: 0.95
-    adam_eps: 1.0e-08
-    name: adamW
-    torch_adam_is_fused: true
-  weight_decay: 0.01
-  zero_stage: 1
-parallelism:
-  dp: 1
-  expert_parallel_size: 1
-  pp: 8
-  pp_engine: 1f1b
-  tp: 8
-  tp_linear_async_communication: false
-  tp_mode: REDUCE_SCATTER
-profiler:
-  profiler_export_path: /fsx/ferdinandmom/ferdinand-hf/bench_cluster/results/llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16
-tokenizer:
-  tokenizer_max_length: null
-  tokenizer_name_or_path: openai-community/gpt2
-  tokenizer_revision: null
-data_stages:
-- name: Training Stage
-  start_training_step: 1
-  data:
-    dataset:
-      dataset_overwrite_cache: false
-      dataset_processing_num_proc_per_process: 64
-      hf_dataset_config_name: null
-      hf_dataset_or_datasets: roneneldan/TinyStories
-      hf_dataset_splits: train
-      text_column_name: text
-    num_loading_workers: 0
-    seed: 42
-lighteval: null
-tokens:
-  train_steps: 20
-  val_check_interval: -1
-  batch_accumulation_per_replica: 64
-  limit_test_batches: 0
-  limit_val_batches: 0
-  micro_batch_size: 16
-  sequence_length: 4096
-logging:
-  iteration_step_info_interval: 1
-  log_level: info
-  log_level_replica: info
-checkpoints:
-  checkpoint_interval: 100000
-  checkpoints_path: /dev/null
-  resume_checkpoint_path: null

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/log_metrics.csv

@@ -1,21 +0,0 @@
-iteration,consumed_tokens,elapsed_time_per_iteration_ms,tokens_per_sec,tokens_per_sec_per_gpu,global_batch_size,lm_loss,lr,model_tflops_per_gpu,hardware_tflops_per_gpu,grad_norm,memory_usage_MiB,peak_allocated_MiB,peak_reserved_MiB
-1,4190000.0000000005,260000.0,16100.000000000002,252.0,1020.0,11.1,0.0001,2.29,2.29,13.4,655.29,34839.89,35196.0
-2,8390000.0,47500.0,88300.0,1380.0,1020.0,11.1,9.53e-05,12.5,12.5,13.5,655.29,34839.89,35196.0
-3,12600000.0,25200.0,166000.0,2600.0,1020.0,9.89,9.05e-05,23.6,23.6,17.3,655.29,34839.89,35196.0
-4,16800000.0,23800.0,176000.0,2760.0,1020.0,10.5,8.58e-05,25.0,25.0,11.9,655.29,655.3,35196.0
-5,21000000.0,20500.0,204000.0,3190.0,1020.0,9.72,8.11e-05,29.0,29.0,9.81,655.29,34839.89,35196.0
-6,25200000.0,21500.0,195000.0,3040.0,1020.0,9.12,7.63e-05,27.6,27.6,5.97,655.29,34839.89,35196.0
-7,29400000.0,20700.0,202000.0,3160.0,1020.0,8.76,7.16e-05,28.7,28.7,5.56,655.29,34839.89,35196.0
-8,33600000.0,20400.0,206000.0,3220.0,1020.0,8.54,6.68e-05,29.2,29.2,6.82,655.29,34839.89,35196.0
-9,37700000.0,18700.0,224000.0,3500.0,1020.0,8.22,6.21e-05,31.7,31.7,4.82,,,
-10,41900000.0,18900.0,222000.0,3470.0,1020.0,8.03,5.74e-05,31.5,31.5,4.81,655.29,34839.89,35196.0
-11,46100000.0,18800.0,223000.0,3480.0,1020.0,7.84,5.26e-05,31.6,31.6,4.56,655.29,34839.89,35196.0
-12,50300000.0,19000.0,221000.0,3450.0,1020.0,7.69,4.79e-05,31.3,31.3,4.35,,,
-13,54500000.0,18300.0,230000.0,3590.0,1020.0,7.54,4.32e-05,32.6,32.6,3.75,655.29,34839.89,35196.0
-14,58700000.0,19000.0,221000.0,3450.0,1020.0,7.42,3.84e-05,31.3,31.3,3.53,,,
-15,62900000.0,18900.0,221000.0,3460.0,1020.0,7.29,3.37e-05,31.4,31.4,3.36,655.29,34839.89,35196.0
-16,67099999.99999999,17900.0,234000.0,3660.0,1020.0,7.2,2.89e-05,33.2,33.2,3.45,655.29,34839.89,35196.0
-17,71300000.0,18000.0,232000.0,3630.0,1020.0,7.12,2.42e-05,33.0,33.0,3.57,655.29,34839.89,35196.0
-18,75500000.0,17500.0,240000.0,3740.0,1020.0,7.04,1.95e-05,34.0,34.0,3.52,655.29,34839.89,35196.0
-19,79700000.0,18600.0,226000.0,3530.0,1020.0,6.98,1.47e-05,32.0,32.0,3.33,,,
-20,83900000.0,18400.0,228000.0,3570.0,1020.0,6.93,1e-05,32.3,32.3,3.08,,,

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/profiler.csv

@@ -1,2 +0,0 @@
-forward,backward
-0ms 897μs,2ms 827μs

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/profiler/ip-26-0-160-225_985827.1720275032620289115.pt.trace.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:29aeb7b23154ccf94c9acbc5a9556af6cdb9b932bd34923cd4690dbe879abcee
-size 1629313426

llama-1B/64_GPUS/dp-1_tp-8_pp-8_mbz-16/status.txt

@@ -1 +0,0 @@
-completed