#!/usr/bin/env bash # Robust HD/SSD/NVMe performance CLI utility # A CrystalDiskMark-style storage benchmark for Linux, built on fio. # Runs the exact same four tests CrystalDiskMark does (SEQ1M Q8T1, SEQ1M Q1T1, # RND4K Q32T1, RND4K Q32T16), each measured for both Read and Write, and prints # the results in a CrystalDiskMark-style table. # Dependencies: fio (apt install fio / pacman -S fio). Falls back to dd if missing. # See: https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance # See: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/ # mReschke 2024-01-18 # CLI Parameters path="$1" option="" engine="" # IO engine; empty = auto-detect (io_uring > libaio > posixaio > sync) direct="" # O_DIRECT; empty = auto-detect; 1 = bypass page cache, 0 = buffered runtime=5 # seconds per measurement (CrystalDiskMark default is 5s) size="1g" # test file size (CrystalDiskMark default is 1GiB) verbose=0 # 1 = also print the full fio output for every run for arg in "${@:2}"; do case "$arg" in --dd) option="--dd" ;; --fio) option="--fio" ;; --slog) option="--slog" ;; --buffered) direct=0 ;; --direct) direct=1 ;; --verbose) verbose=1 ;; --engine=*) engine="${arg#*=}" ;; --direct=*) direct="${arg#*=}" ;; --runtime=*) runtime="${arg#*=}" ;; --size=*) size="${arg#*=}" ;; esac done # Main application flow function main { # Show usage if no params if [ ! "$path" ]; then usage fi # Understand . path if [ "$path" == '.' ]; then path=$(pwd) fi # Check if path exists if [ ! -e "$path" ]; then echo "Path $path does not exist" exit 1 fi # Must type y or n THEN press enter (which I like better) echo "NOTICE: ${size^^} free space on '$path' is required to perform the benchmark." echo -n "Are you ready to start a storage benchmark against '$path' ? "; read answer if [ "$answer" != "${answer#[Yy]}" ]; then echo "Great! Starting benchmark now!" else echo "Ok, cancelled!" exit 0 fi # Use dd or fio based on param or defaults if [ "$option" == "--dd" ]; then dd_speedtest elif [ "$option" == "--fio" ]; then cdm_speedtest elif [ "$option" == "--slog" ]; then slog_speedtest elif [ "$option" == "" ]; then # If fio is installed, use it, else use dd if ! command -v fio &> /dev/null; then echo "" echo "fio is not installed -- falling back to basic dd test." echo "Install fio for the full CrystalDiskMark-style benchmark." dd_speedtest else cdm_speedtest fi fi } function fio_probe { # Run a tiny throwaway fio job to see if a given (engine, direct) combo # actually works on this path/filesystem. Returns 0 on success. This is how # we stay accurate AND portable across ext4, xfs, btrfs, ZFS, NFS, etc. # without the caller needing to know what each filesystem supports. local eng="$1" dir="$2" sudo fio --name=probe \ --directory="$path" \ --ioengine="$eng" \ --rw=write --bs=4k --size=1m \ --direct="$dir" \ --time_based --runtime=1 \ > /dev/null 2>&1 local rc=$? rm -rf "$path"/probe* 2>/dev/null return $rc } function detect_io_settings { # Pick the fastest available IO engine unless the user forced one with # --engine=. io_uring is the modern, lowest-overhead Linux engine; libaio # is older (and only truly async with O_DIRECT); posixaio/sync are the # portable fallbacks (eg some NFS mounts). if [ -z "$engine" ]; then for eng in io_uring libaio posixaio sync; do if fio_probe "$eng" 0; then engine="$eng"; break; fi done [ -z "$engine" ] && engine="sync" fi # Decide O_DIRECT unless forced with --direct/--buffered. O_DIRECT bypasses # the OS page cache so we measure the device instead of RAM. Not every # filesystem supports it (eg older OpenZFS < 2.3, some NFS mounts), so we # probe and fall back to buffered rather than erroring out. if [ -z "$direct" ]; then if fio_probe "$engine" 1; then direct=1; else direct=0; fi fi } function fio_bw_mbps { # Extract bandwidth in decimal MB/s (matching CrystalDiskMark) from a fio # run. fio prints eg " READ: bw=3357MiB/s (3521MB/s), ..." -- we take the # parenthetical SI figure and normalise kB/MB/GB into MB/s. local out="$1" local raw num unit raw=$(echo "$out" | /usr/bin/grep -oP 'bw=\S+\s+\(\K[^)]+' | head -n1) [ -z "$raw" ] && { echo "0.00"; return; } num=$(echo "$raw" | /usr/bin/grep -oP '[0-9.]+' | head -n1) unit=$(echo "$raw" | /usr/bin/grep -oP '[A-Za-z]+/s' | head -n1) case "$unit" in B/s) awk -v n="$num" 'BEGIN{printf "%.2f", n/1000000}' ;; kB/s|KB/s) awk -v n="$num" 'BEGIN{printf "%.2f", n/1000}' ;; MB/s) awk -v n="$num" 'BEGIN{printf "%.2f", n}' ;; GB/s) awk -v n="$num" 'BEGIN{printf "%.2f", n*1000}' ;; *) awk -v n="$num" 'BEGIN{printf "%.2f", n}' ;; esac } function fio_iops { # Extract IOPS from a fio run. fio prints eg " read: IOPS=12.3k, BW=..." -- # the value may carry a k/M suffix, which we expand to a whole number. local out="$1" local raw num unit raw=$(echo "$out" | /usr/bin/grep -oP 'IOPS=\K[0-9.]+[kKmM]?' | head -n1) [ -z "$raw" ] && { echo "0"; return; } num=$(echo "$raw" | /usr/bin/grep -oP '[0-9.]+' | head -n1) unit=$(echo "$raw" | /usr/bin/grep -oP '[kKmM]' | head -n1) case "$unit" in k|K) awk -v n="$num" 'BEGIN{printf "%.0f", n*1000}' ;; m|M) awk -v n="$num" 'BEGIN{printf "%.0f", n*1000000}' ;; *) awk -v n="$num" 'BEGIN{printf "%.0f", n}' ;; esac } function run_fio { # run_fio -> echoes "MB/s IOPS" (space-sep) # A single shared 1GB test file is reused across every run (laid out once), # which keeps the footprint at one file and avoids re-creating it each time. local rw="$1" bs="$2" qd="$3" jobs="$4" local args=( --name=speedtest --filename="$benchfile" --ioengine="$engine" --direct="$direct" --rw="$rw" --bs="$bs" --size="$size" --numjobs="$jobs" --iodepth="$qd" --time_based --runtime="$runtime" --group_reporting=1 ) # Flush device cache at the end of write tests so cached writes can't lie. case "$rw" in *write) args+=(--end_fsync=1) ;; esac # Capture the full fio output. In verbose mode we also keep fio's stderr # (2>&1); otherwise we discard it. Either way only the parsed bandwidth is # echoed to stdout, so the captured value stays clean. local out if [ "$verbose" = 1 ]; then out=$(sudo fio "${args[@]}" 2>&1) else out=$(sudo fio "${args[@]}" 2>/dev/null) fi # Dump the raw fio output (to stderr, so it never pollutes the captured # values on stdout) when running verbose. if [ "$verbose" = 1 ]; then echo " --- fio: $rw bs=$bs iodepth=$qd numjobs=$jobs ---" >&2 echo "$out" >&2 echo "" >&2 fi echo "$(fio_bw_mbps "$out") $(fio_iops "$out")" } # Table drawing ------------------------------------------------------------- # Label cell is "| %-16s |" (18 dashes); the four numeric cells are "| %14s |" # (16 dashes each). TBL_SEG_LBL="------------------" # 18 dashes -> matches "| %-16s |" TBL_SEG_NUM="----------------" # 16 dashes -> matches "| %14s |" function tbl_rule { printf "+%s+%s+%s+%s+%s+\n" "$TBL_SEG_LBL" "$TBL_SEG_NUM" "$TBL_SEG_NUM" "$TBL_SEG_NUM" "$TBL_SEG_NUM" } function tbl_row { # tbl_row # (col1 left-aligned, the four numeric cols right-aligned) printf "| %-16s | %14s | %14s | %14s | %14s |\n" "$1" "$2" "$3" "$4" "$5" } # Five-column table for the --slog latency profile. # "| %-16s |" -> 18 dashes ; "| %12s |" -> 14 dashes SLOG_SEG16="------------------" # 18 dashes SLOG_SEG12="--------------" # 14 dashes function slog_rule { printf "+%s+%s+%s+%s+%s+\n" "$SLOG_SEG16" "$SLOG_SEG12" "$SLOG_SEG12" "$SLOG_SEG12" "$SLOG_SEG12" } function slog_row { # slog_row (col1 left, rest right-aligned) printf "| %-16s | %12s | %12s | %12s | %12s |\n" "$1" "$2" "$3" "$4" "$5" } function run_fio_sync { # run_fio_sync # -> echoes "IOPS MB/s p50_us p99_us mean_us" (one space-separated line) # Forces synchronous IO (--sync=1 => O_SYNC) with the portable psync engine so # every write is a ZIL commit -- i.e. it exercises the SLOG exactly the way a # sync=always dataset (NFS/iSCSI/VM) does, regardless of the dataset's own # sync property. We deliberately DON'T use --group_reporting here: each job is # reported independently in the JSON and we aggregate ourselves (sum IOPS/bw, # average p50, take the worst p99), which sidesteps fio's group-merge quirks. local bs="$1" jobs="$2" local args=( --name=slog --filename="$benchfile" --ioengine=psync --rw=randwrite --bs="$bs" --size="$size" --numjobs="$jobs" --sync=1 --time_based --runtime="$runtime" --output-format=json ) local out out=$(sudo fio "${args[@]}" 2>/dev/null) if [ "$verbose" = 1 ]; then echo " --- fio json: randwrite bs=$bs numjobs=$jobs sync=1 ---" >&2 echo "$out" >&2 echo "" >&2 fi # Parse the JSON: sum IOPS and bandwidth across jobs, aggregate completion # latency percentiles (clat is in ns -> convert to us). bw_bytes is bytes/s. echo "$out" | python3 -c ' import json, sys try: d = json.load(sys.stdin) jobs = d["jobs"] except Exception: print("0 0.00 0.0 0.0 0.0"); sys.exit(0) iops = sum(j["write"]["iops"] for j in jobs) mbps = sum(j["write"]["bw_bytes"] for j in jobs) / 1e6 def pctl(j, p): return j["write"]["clat_ns"]["percentile"].get(p) p50 = [pctl(j, "50.000000") for j in jobs if pctl(j, "50.000000") is not None] p99 = [pctl(j, "99.000000") for j in jobs if pctl(j, "99.000000") is not None] mean = [j["write"]["clat_ns"]["mean"] for j in jobs] p50_us = (sum(p50)/len(p50))/1000.0 if p50 else 0.0 # average of per-job medians p99_us = (max(p99))/1000.0 if p99 else 0.0 # worst-case tail across jobs mean_us = (sum(mean)/len(mean))/1000.0 if mean else 0.0 print("%.0f %.2f %.1f %.1f %.1f" % (iops, mbps, p50_us, p99_us, mean_us)) ' } function slog_speedtest { # SLOG / sync-write latency profile. Detect engine/direct only for the banner; # the actual runs always force psync + O_SYNC (see run_fio_sync). detect_io_settings benchfile="$path/speedtest-hd.bench" if ! command -v fio &> /dev/null; then echo "ERROR: --slog requires fio (apt install fio / pacman -S fio)." >&2 exit 1 fi if ! command -v python3 &> /dev/null; then echo "ERROR: --slog requires python3 (used to parse fio JSON latency percentiles)." >&2 exit 1 fi echo "" echo " speedtest-hd : SLOG / sync-write latency profile" echo " Target : $path" echo " Method : fio randwrite bs=4k --sync=1 (O_SYNC), psync engine" echo " Profile : runtime=${runtime}s/run size=${size^^}" echo " Note : every write is a synchronous ZIL commit -- this is the load" echo " your SLOG actually sees. Watch it live in another shell with:" echo " zpool iostat -vl 1" echo "" # T1 is the headline single-stream latency; the sweep shows how the SLOG # scales as concurrent sync writers (NFS/iSCSI/VM threads) pile on. Run every # measurement FIRST (progress goes to stderr), collect the results, then draw # the whole table in one block so the "measuring..." lines can't interleave # into the middle of the table. local jobs_list="1 4 8 16" local -a rows local j res iops bw p50 p99 mean for j in $jobs_list; do echo " measuring 4K sync randwrite T$j ..." >&2 res=$(run_fio_sync 4k "$j") read -r iops bw p50 p99 mean <<< "$res" rows+=("$(slog_row "4K sync T$j" "$iops" "$bw" "$p50" "$p99")") done rm -f "$benchfile" # Render the assembled table echo "" slog_rule slog_row "Test" "IOPS" "MB/s" "p50 lat(us)" "p99 lat(us)" slog_rule printf '%s\n' "${rows[@]}" slog_rule echo "" echo " Healthy Optane SLOG (eg P1600X) single-stream (T1) target:" echo " ~15-25k IOPS, p50 latency ~40-65us. Much higher latency usually means" echo " CPU C-states / PCIe ASPM / BIOS power profile (eg Dell DAPC) throttling." echo "" } function cdm_speedtest { # Auto-detect the best engine and whether O_DIRECT works on this path. detect_io_settings benchfile="$path/speedtest-hd.bench" local direct_label="enabled (device)" [ "$direct" != 1 ] && direct_label="DISABLED (buffered -- may reflect RAM cache!)" # Banner echo "" echo " speedtest-hd : CrystalDiskMark-style storage benchmark" echo " Target : $path" echo " Engine : $engine O_DIRECT: $direct_label" echo " Profile : size=${size^^} runtime=${runtime}s/run (8 runs)" echo "" # The four CrystalDiskMark default tests. Q = queue depth (--iodepth), # T = threads (--numjobs). The first three are T1 (single thread); the last # is T16. A single shared test file is reused -- with numjobs>1 every job # issues IO against the same file, which is fine. # Each run now returns both MB/s and IOPS ("bw iops"), so we capture the pair. # label read-rw write-rw bs Q T echo " measuring SEQ1M Q8T1 ..." >&2 local s8r_bw s8r_io; read -r s8r_bw s8r_io <<< "$(run_fio read 1m 8 1)" local s8w_bw s8w_io; read -r s8w_bw s8w_io <<< "$(run_fio write 1m 8 1)" echo " measuring SEQ1M Q1T1 ..." >&2 local s1r_bw s1r_io; read -r s1r_bw s1r_io <<< "$(run_fio read 1m 1 1)" local s1w_bw s1w_io; read -r s1w_bw s1w_io <<< "$(run_fio write 1m 1 1)" echo " measuring RND4K Q32T1 ..." >&2 local r32r_bw r32r_io; read -r r32r_bw r32r_io <<< "$(run_fio randread 4k 32 1)" local r32w_bw r32w_io; read -r r32w_bw r32w_io <<< "$(run_fio randwrite 4k 32 1)" echo " measuring RND4K Q32T16 ..." >&2 local r1r_bw r1r_io; read -r r1r_bw r1r_io <<< "$(run_fio randread 4k 32 16)" local r1w_bw r1w_io; read -r r1w_bw r1w_io <<< "$(run_fio randwrite 4k 32 16)" # Cleanup the shared test file rm -f "$benchfile" # Render the CrystalDiskMark-style table echo "" tbl_rule tbl_row "Test" "Read (MB/s)" "Write (MB/s)" "Read (IOPS)" "Write (IOPS)" tbl_rule tbl_row "SEQ1M Q8T1" "$s8r_bw" "$s8w_bw" "$s8r_io" "$s8w_io" tbl_row "SEQ1M Q1T1" "$s1r_bw" "$s1w_bw" "$s1r_io" "$s1w_io" tbl_row "RND4K Q32T1" "$r32r_bw" "$r32w_bw" "$r32r_io" "$r32w_io" tbl_row "RND4K Q32T16" "$r1r_bw" "$r1w_bw" "$r1r_io" "$r1w_io" tbl_rule echo "" } function dd_speedtest { # Basic HD speed test using dd (fallback when fio is not installed) # mReschke 2017-07-11 local file ddsize file=$path/bigfile ddsize=1024 echo "Running dd based HD/SSD/NVMe Benchmarks" echo "---------------------------------------" printf "Cached write speed...\n" dd if=/dev/zero of=$file bs=1M count=$ddsize printf "\nUncached write speed...\n" dd if=/dev/zero of=$file bs=1M count=$ddsize conv=fdatasync,notrunc printf "\nUncached read speed...\n" echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null dd if=$file of=/dev/null bs=1M count=$ddsize printf "\nCached read speed...\n" dd if=$file of=/dev/null bs=1M count=$ddsize rm $file printf "\nDone\n" } # Show help and usage information function usage { echo "speedtest-hd : CrystalDiskMark-style storage benchmark" echo " Runs the same four tests as CrystalDiskMark (each Read + Write):" echo " SEQ1M Q8T1 Sequential 1MiB, queue depth 8, 1 thread" echo " SEQ1M Q1T1 Sequential 1MiB, queue depth 1, 1 thread" echo " RND4K Q32T1 Random 4KiB, queue depth 32, 1 thread" echo " RND4K Q32T16 Random 4KiB, queue depth 32, 16 threads" echo " Uses fio if installed (recommended), else basic dd." echo "mReschke 2024-01-18" echo "" echo "NOTICE: creates one test file (default 1GB) on the destination disk." echo "Please ensure you have write access with enough free space." echo "" echo "Usage:" echo " Auto (fio if installed, else dd):" echo " ./speedtest-hd /mnt/somedisk" echo " ./speedtest-hd ." echo "" echo " Force fio / force dd:" echo " ./speedtest-hd /mnt/somedisk --fio" echo " ./speedtest-hd /mnt/somedisk --dd" echo "" echo " SLOG / sync-write latency profile (ZFS ZIL, NFS/iSCSI/VM sync IO):" echo " ./speedtest-hd /mnt/zpool/dataset --slog" echo " 4K synchronous writes at T1/T4/T8/T16; reports IOPS, MB/s, p50/p99" echo " commit latency. Requires fio + python3. (Tip: watch the SLOG with" echo " 'zpool iostat -vl 1' in another shell.)" echo "" echo " Tuning flags (auto-detected by default, override as needed):" echo " --engine=io_uring|libaio|posixaio|sync IO engine (default: auto)" echo " --direct Force O_DIRECT (bypass page cache)" echo " --buffered Force buffered IO (eg if O_DIRECT is unsupported)" echo " --runtime=SEC Seconds per run (default: 5, like CrystalDiskMark)" echo " --size=SIZE Test file size (default: 1g)" echo " --verbose Print full fio output for every run (summary table unchanged)" echo " Examples:" echo " ./speedtest-hd /mnt/nvmepool --runtime=10 --size=4g" echo " ./speedtest-hd /mnt/nfsshare --buffered" echo " ./speedtest-hd /mnt/nvmepool --verbose" echo " ./speedtest-hd /mnt/nvme-ultra-r10/vm-root --slog --runtime=30" exit 0 } # Go main