Skip to content
Snippets Groups Projects
Commit fc9e17e5 authored by Xuan Gu's avatar Xuan Gu
Browse files

Update 2 files

- /scripts/benchmark_sbatch_submit.sh
- /README.md
parent b47420dc
No related branches found
No related tags found
No related merge requests found
...@@ -59,14 +59,16 @@ bash scripts/copy_data_for_benchmark.sh ...@@ -59,14 +59,16 @@ bash scripts/copy_data_for_benchmark.sh
You can choose either singularity or enroot in the scripts ```benchmark_single_node.sbatch``` and ```benchmark_multi_node.sbatch```. You can choose either singularity or enroot in the scripts ```benchmark_single_node.sbatch``` and ```benchmark_multi_node.sbatch```.
The input arguments are: The input arguments are:
1. Number of nodes, 1. Data dimention,
2. Number of gpus per node, 2. Number of nodes,
3. Number of iterations for each parameter setting. 3. Number of gpus per node,
4. Number of iterations for each parameter setting.
5. Batch size
We will average the benchmark performance over the iterations. The maximum usable (without a OOM error) batch size is 256 and 128 for single and multi-node, respectively. We will average the benchmark performance over the iterations. The maximum usable (without a OOM error) batch size is 256 and 128 for single and multi-node, respectively.
``` ```
cd cd Berzelius-nnU-Net-Benchmark && mkdir -p sbatch_out cd cd Berzelius-nnU-Net-Benchmark && mkdir -p sbatch_out
bash scripts/benchmark_sbatch_submit.sh 1 8 100 128 bash scripts/benchmark_sbatch_submit.sh 2 1 8 10 128
``` ```
### Results ### Results
......
#!/bin/bash #!/bin/bash
mkdir -p sbatch_out NUM_NODES=$2
NUM_GPUS=$3
NUM_ITERATIONS=$4
dim=2 dim=$1
NUM_NODES=$1 batch_size=$5
NUM_GPUS=$2
NUM_ITERATIONS=$3
BATCH_SIZE=$4
if [ $NUM_NODES -eq 1 ]; then if [ $NUM_NODES -eq 1 ]; then
for nodes in {1..$((NUM_NODES))}; do for ((nodes = 1; nodes <= $NUM_NODES; nodes++)); do
for gpus in {1..$((NUM_GPUS))}; do for ((gpus = 1; gpus <= $NUM_GPUS; gpus++)); do
for batch_size in $((BATCH_SIZE)); do for ((iteration = 1; iteration <= $NUM_ITERATIONS; iteration++)); do
for iteration in {1..$((NUM_ITERATIONS))}; do
echo dim ${dim}, nodes ${nodes}, gpus ${gpus}, batch_size ${batch_size}, iteration ${iteration}
# For single node
#sbatch -o sbatch_out/benchmark_dim${dim}_nodes${nodes}_gpus${gpus}_batchsize${batch_size}_iteration${iteration}.out scripts/benchmark_single_node.sbatch ${dim} ${nodes} ${gpus} ${batch_size} ${iteration}
echo dim ${dim}, nodes ${nodes}, gpus ${gpus}, batch_size ${batch_size}, iteration ${iteration} sleep 1 # pause to be kind to the scheduler
# For single node
#sbatch -o sbatch_out/benchmark_dim${dim}_nodes${nodes}_gpus${gpus}_batchsize${batch_size}_iteration${iteration}.out scripts/benchmark_single_node.sbatch ${dim} ${nodes} ${gpus} ${batch_size} ${iteration}
sleep 1 # pause to be kind to the scheduler
done
done done
done done
done done
else else
for nodes in {2..$((NUM_NODES))}; do for ((nodes = 2; nodes <= $NUM_NODES; nodes++)); do
for gpus in {$((NUM_GPUS))}; do for ((gpus = 8; gpus <= $NUM_GPUS; gpus++)); do
for batch_size in $((BATCH_SIZE)); do for ((iteration = 1; iteration <= $NUM_ITERATIONS; iteration++)); do
for iteration in {1..$((NUM_ITERATIONS))}; do batch_size=$BATCH_SIZE
# For multi node
# For multi node sbatch -o sbatch_out/benchmark_dim${dim}_nodes${nodes}_gpus${gpus}_batchsize${batch_size}_iteration${iteration}.out scripts/benchmark_multi_node.sbatch ${dim} ${nodes} ${gpus} ${batch_size} ${iteration}
sbatch -o sbatch_out/benchmark_dim${dim}_nodes${nodes}_gpus${gpus}_batchsize${batch_size}_iteration${iteration}.out scripts/benchmark_multi_node.sbatch ${dim} ${nodes} ${gpus} ${batch_size} ${iteration}
sleep 1 # pause to be kind to the scheduler sleep 1 # pause to be kind to the scheduler
done
done done
done done
done done
fi fi
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment