cwm/eval/Flow/create_spring_submission_parallel.sh

#!/bin/bash

# Define the path to the dataset and the Python script
DATASET_PATH="/ccn2/dataset/Flows_Kinetics/SPRING/spring/test/"
SCRIPT_PATH="./create_spring_submission_unified.py"
SAVE_DATA_PATH=${1}
MODEL=${2}
# Counter for GPUs
GPU_COUNTER=0

# Number of GPUs available
NUM_GPUS=8

#kill session
tmux kill-session -t extraction

tmux new-session -d -s "extraction"

# Iterate through each folder in the dataset
for FOLDER in $(find $DATASET_PATH -mindepth 1 -maxdepth 1 -type d | sort); do
    # Extract the folder name for the tmux session name
    FOLDER_NAME=$(basename $FOLDER)

    # Create a new detached tmux session for each folder
    tmux new-window -t extraction -n "$FOLDER" "ulimit -n 65535; CUDA_VISIBLE_DEVICES=$GPU_COUNTER python $SCRIPT_PATH --folder $FOLDER --gpu $GPU_COUNTER --save_data_path $SAVE_DATA_PATH --model $MODEL; echo 'Press Enter to continue...'; read -p ''"
    # Increment the GPU counter and reset if it exceeds the number of GPUs
    GPU_COUNTER=$((GPU_COUNTER + 1))
    if [ $GPU_COUNTER -ge $NUM_GPUS ]; then
        GPU_COUNTER=0
    fi

    sleep 1
done

tmux attach-session -t extraction