submit_benchmark_jobs.sh

#!/bin/bash
set -e

WORK_DIR=/proj/nsc_testing/xuan/berzelius-benchmarks/NVIDIA/DeepLearningExamples/PyTorch/Segmentation/nnUNet
benchmark_modes=("train" "predict")
node_types=("thin" "fat")

dim=2
for nodes in {1..1}; do
    for gpus in {1..8}; do
        for benchmark_mode in "${benchmark_modes[@]}"; do
            for node_type in "${node_types[@]}"; do
        

                    if [ "${node_type}" = "thin" ]; then
                        batch_size=512
                    else
                        batch_size=1024
                    fi

                    echo dim ${dim}, nodes ${nodes}, gpus ${gpus}, batch_size ${batch_size}, benchmark_mode ${benchmark_mode}, node_type ${node_type}

                    # For single node
                    bash $WORK_DIR/generate_benchmark_jobs.sh ${dim} ${nodes} ${gpus} ${batch_size} ${benchmark_mode} ${node_type}
                    SBATCH_DIR=$WORK_DIR/sbatch_scripts/benchmark_${node_type}_${benchmark_mode}_dim${dim}_nodes${nodes}_gpus${gpus}_batchsize_${batch_size}.sbatch
                    sbatch $SBATCH_DIR
                    sleep 180 
            done
        done
    done
done