From fcc5a2f8f60cfe6d7a2d2c8a00000413a99f84ac Mon Sep 17 00:00:00 2001
From: Sebastian Thorarensen <sebth@nsc.liu.se>
Date: Wed, 25 Sep 2024 23:20:59 +0200
Subject: [PATCH] Write instruction
---
README | 153 ++++++++++++++++++++++++++++++++++++++-
storage-benchmark.sbatch | 29 ++++----
2 files changed, 166 insertions(+), 16 deletions(-)
diff --git a/README b/README
index f998079..989d052 100644
--- a/README
+++ b/README
@@ -1 +1,152 @@
-Write instructions here, with and without Slurm!
+Arrhenius Storage Benchmark
+===========================
+
+elbencho 3.0.19 <https://github.com/breuner/elbencho/releases/tag/v3.0-19>
+will be used for the benchmark.
+
+The elbencho service will be started on all client nodes participating
+in the test by running
+
+ elbencho --service --foreground
+
+on each client node.
+
+
+Sequential read and write
+-------------------------
+
+After the elbencho service has been started on all client nodes
+participating in the benchmark, the sequential read and write part of
+the benchmark will be started by running
+
+ elbencho --hosts <hosts> --rotatehosts=1 -t <threads per node> \
+ -w -r -b <blocksize> -s <bytes per thread> -n 0 -F <directory>
+
+where
+
+ - <hosts> is a comma-separated list of the participating client nodes'
+ hostnames,
+
+ - <threads per node> is the number of threads (tasks) per client node
+ performing I/O,
+
+ - <blocksize> is the number of bytes written or read in a single
+ operation,
+
+ - <bytes per thread> is the number of bytes written and read per
+ thread (task),
+
+ - <directory> is the path to a directory on the filesystem being
+ benchmarked.
+
+<threads per node> and <blocksize> will be chosen optimally for best
+performance. <bytes per thread> will be chosen so that the write and
+read phases run for up to 60 minutes each.
+
+Results will be evaluated by looking at the "WRITE" and "READ" sections
+in the table output from elbencho on stdout. The row "Elapsed time" in
+column "LAST DONE" in each section will be used to evaluate how long a
+phase ran for. The row "Throughput MiB/s" in column "LAST DONE" in each
+section will be used to evaluate the sequential write and read
+performance.
+
+
+IOPS
+----
+
+After the elbencho service has been started on all client nodes
+participating in the benchmark, the IOPS part of the benchmark will be
+started by running
+
+ elbencho --hosts <hosts> --rotatehosts=1 -t <threads per node> \
+ -w -r -b 4K -s <bytes per thread> -n 0 -F --rand <directory>
+
+where
+
+ - <hosts> is a comma-separated list of the participating client nodes'
+ hostnames,
+
+ - <threads per node> is the number of threads (tasks) per client node
+ performing I/O,
+
+ - <bytes per thread> is the number of bytes written and read per
+ thread (task),
+
+ - <directory> is the path to a directory on the filesystem being
+ benchmarked.
+
+<threads per node> will be chosen optimally for best performance.
+<bytes per thread> will be chosen so that the write and read phases runs
+for up to 60 minutes each.
+
+Results will be evaluated by looking at the "WRITE" and "READ" sections
+in the table output from elbencho on stdout. The row "Elapsed time" in
+column "LAST DONE" in each section will be used to evaluate how long a
+phase ran for. The row "IOPS" in column "LAST DONE" in each section
+will be used to evaluate the write and read IOPS.
+
+
+Metadata
+--------
+
+After the elbencho service has been started on all client nodes
+participating in the benchmark, the metadata part of the benchmark will
+be started by running
+
+ elbencho --hosts <hosts> --rotatehosts=1 -t <threads per node> \
+ -d -w --stat -F -N <files per thread> -D <directory>
+
+where
+
+ - <hosts> is a comma-separated list of the participating client nodes'
+ hostnames,
+
+ - <threads per node> is the number of threads (tasks) per client node
+ performing I/O,
+
+ - <files per thread> is the number of files created, stat, and deleted
+ per thread (task),
+
+ - <directory> is the path to a directory on the filesystem being
+ benchmarked.
+
+<threads per node> will be chosen optimally for best performance.
+<number of files> will be chosen so that each phase; create, stat, and
+delete; runs for up to 60 minutes.
+
+Results will be evaluated by looking at the "WRITE", "STAT", and
+"RMFILES" sections in the table output from elbencho on stdout. The
+"WRITE" section shows the "create" performance. The "STAT" section
+shows the "stat" performance". The "RMFILES" shows the "delete"
+performance.
+
+The row "Elapsed time" in column "LAST DONE" in each section will be
+used to evaluate how long a phase ran for. The row "Files/s" in column
+"LAST DONE" in each section will be used to evaluate the number of
+creates, stats, and deletes per second.
+
+
+sbatch script
+=============
+
+For convenience, a Slurm sbatch script is included in this repository.
+This section describes how to run the different parts of the benchmark
+using the sbatch script.
+
+Sequential read and write:
+
+ sbatch -N <number of nodes> --cpus-per-task=<threads per node> \
+ storage-benchmark.sbatch \
+ sequential <blocksize> <bytes per thread> <directory>
+
+IOPS:
+
+ sbatch -N <number of nodes> --cpus-per-task=<threads per node> \
+ storage-benchmark.sbatch \
+ iops <bytes per thread> <directory>
+
+Metadata:
+
+ sbatch -N <number of nodes> --cpus-per-task=<threads per node> \
+ storage-benchmark.sbatch \
+ meta <files per thread> <directory>
diff --git a/storage-benchmark.sbatch b/storage-benchmark.sbatch
index 616237e..15c5525 100644
--- a/storage-benchmark.sbatch
+++ b/storage-benchmark.sbatch
@@ -2,7 +2,7 @@
#SBATCH -J arrh-storage-benchmark
usage() {
- echo "Usage: sbatch -N <number of nodes> --cpus-per-task=<threads per node> storage-benchmark.sbatch [ stream <blocksize> | iops | meta ] <directory>"
+ echo "Usage: sbatch -N <number of nodes> --cpus-per-task=<threads per node> storage-benchmark.sbatch [ sequential <blocksize> | iops | meta ] <bytes or files per thread> <directory>"
exit 2
}
@@ -15,7 +15,7 @@ info() {
##
MODE=$1
-if [ "$MODE" = stream ]
+if [ "$MODE" = sequential ]
then
BLOCKSIZE=$2
shift
@@ -24,7 +24,12 @@ then :
else usage
fi
-DIRECTORY=$2
+BYTES_OR_FILES=$2
+if [ -z "$BYTES_OR_FILES" ]
+then usage
+fi
+
+DIRECTORY=$3
if [ -z "$DIRECTORY" ]
then usage
fi
@@ -35,7 +40,7 @@ HOSTS=$(scontrol show hostnames | tr '\n' ',')
info "Mode: $MODE"
if [ "$BLOCKSIZE" ]
-then info "Block size: $BLOCKSIZE"
+then info "Blocksize: $BLOCKSIZE"
fi
info "Number of nodes: $NNODES"
info "Threads per node: $THREADS"
@@ -47,34 +52,28 @@ elbencho --version
info "Starting service on all nodes"
srun --ntasks-per-node=1 elbencho --service --foreground > /dev/null &
-sleep 5 # wait for services to start
+sleep 10 # wait for services to start
info "Starting storage benchmark"
echo
-if [ "$MODE" = stream ]
+if [ "$MODE" = sequential ]
then
- # 1024 GiB per node
- SIZE=$((1 * 1024 * 1024 * 1024 / THREADS))
elbencho --hosts "$HOSTS" --rotatehosts=1 -t "$THREADS" \
- -w -r -s "$SIZE"K -b "$BLOCKSIZE" -n 0 -F "$DIRECTORY"
+ -w -r -b "$BLOCKSIZE" -s "$BYTES_OR_FILES" -n 0 -F "$DIRECTORY"
elif [ "$MODE" = iops ]
then
- # 128 GiB per node
- SIZE=$((128 * 1024 * 1024 / THREADS))
elbencho --hosts "$HOSTS" --rotatehosts=1 -t "$THREADS" \
- -w -r -s "$SIZE"K -b 4K -n 0 -F --rand "$DIRECTORY"
+ -w -r -b 4K -s "$BYTES_OR_FILES" -n 0 -F --rand "$DIRECTORY"
elif [ "$MODE" = meta ]
then
- # 10M files per node
- FILES=$((10000000 / THREADS))
elbencho --hosts "$HOSTS" --rotatehosts=1 -t "$THREADS" \
- -d -w --stat -F -N "$FILES" -D "$DIRECTORY"
+ -d -w --stat -F -N "$BYTES_OR_FILES" -D "$DIRECTORY"
fi
--
GitLab