mreschke/speedtest-hd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Claude pass, based mods to queuedepth and params

Browse Source
This commit is contained in:
Matthew Reschke
2026-06-21 17:34:17 -06:00
parent ac9cd25cb3
commit 4417a668bc
1 changed files with 149 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
speedtest-hd.sh
+149 -63
View File
@@ -11,11 +11,19 @@
path="$1" path="$1"
option="" option=""
simple=false simple=false
engine="" # IO engine; empty = auto-detect (io_uring > libaio > posixaio > sync)
direct="" # O_DIRECT; empty = auto-detect; 1 = bypass page cache, 0 = buffered
runtime=30 # seconds per test (longer = more accurate, exposes SLC cache exhaustion)
for arg in "${@:2}"; do for arg in "${@:2}"; do
case "$arg" in case "$arg" in
--simple) simple=true ;; --simple) simple=true ;;
--dd) option="--dd" ;; --dd) option="--dd" ;;
--fio) option="--fio" ;; --fio) option="--fio" ;;
--buffered) direct=0 ;;
--direct) direct=1 ;;
--engine=*) engine="${arg#*=}" ;;
--direct=*) direct="${arg#*=}" ;;
--runtime=*) runtime="${arg#*=}" ;;
esac esac
done done
@@ -86,50 +94,52 @@ function print_result {
} }
function fio_write_single_random_4k { function fio_write_single_random_4k {
# Single 4k Random Writes # 4K Random Writes (parallel, deep queue)
# This is a single process doing random 4K writes. This is where the pain # Random 4K writes are the worst thing you can ask a disk to do. Where this
# really, really lives; it's basically the worst possible thing you can ask a # happens most in real life: copying home directories and dotfiles, email
# disk to do. Where this happens most frequently in real life: copying home # stores, some database operations, source code trees.
# directories and dotfiles, manipulating email stuff, some database operations,
# source code trees.
# When I ran this test against the high-performance SSDs in my Ubuntu # IMPORTANT: the original version of this test used numjobs=1 + iodepth=1.
# workstation, they pushed 127MiB/sec. The server just beneath it in the rack # That measures single-op LATENCY (the absolute worst case): the OS issues
# only managed 33MiB/sec on its "high-performance" 7200RPM rust disks... but # one 4K write, waits for it to be acknowledged, then issues the next. On
# even then, the vast majority of that speed is because the data is being # NVMe that lands around ~3000 IOPS / ~12MB/sec -- which looks alarmingly
# written asynchronously, allowing the operating system to batch it up into # slow but is NOT a bug. NVMe's entire advantage is deep command queues and
# larger, more efficient write operations. # parallelism, so a queue depth of 1 deliberately measures the one scenario
# NVMe is bad at. To measure the drive's actual capability we now drive it
# with multiple jobs and a deep queue (4 jobs x iodepth=64), keeping
# hundreds of operations in flight at once. Expect this to jump into the
# hundreds of MB/sec (often 1GB/sec+) on a healthy NVMe pool.
# If we add the argument --fsync=1, forcing the operating system to perform # --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the directory you point it at.
# synchronous writes (calling fsync after each block of data is written) the # --directory= where the test files are created; this is the path/mount you are benchmarking.
# picture gets much more grim: 2.6MiB/sec on the high-performance SSDs but # --ioengine=$engine the engine fio uses to talk to the kernel, auto-detected by detect_io_settings(). On modern Linux this is io_uring (lowest overhead); it falls back to libaio, then posixaio, then sync. NOTE: libaio is only truly asynchronous with --direct=1; with buffered IO it silently degrades to synchronous, which is exactly why io_uring is preferred.
# only 184KiB/sec on the "high-performance" rust. The SSDs were about four # --rw=randwrite random write operations. Other options include read, write (sequential), randread, and randrw.
# times faster than the rust when data was written asynchronously but a # --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk and adds a ton of command-channel overhead, since a separate operation has to be commanded for each 4K of data.
# whopping fourteen times faster when # --size=256m each job's test file is 256MB. With --numjobs=4 that's 4 files totaling ~1GB.
# --numjobs=4 run 4 parallel processes, each with its own test file, to feed the drive from multiple threads at once (single-threaded cannot saturate NVMe).
# --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the working directory you're currently in. # --iodepth=64 how many commands each job keeps stacked in the queue at once. Deep queues are how NVMe reaches its real IOPS; 4 jobs x 64 = up to 256 ops in flight.
# --ioengine=posixaio sets the mode fio interacts with the filesystem. POSIX is a standard Windows, Macs, Linux, and BSD all understand, so it's great for portability—although inside fio itself, Windows users need to invoke --ioengine=windowsaio, not --ioengine=posixaio, unfortunately. AIO stands for Asynchronous Input Output and means that we can queue up multiple operations to be completed in whatever order the OS decides to complete them. (In this particular example, later arguments effectively nullify this.) # --direct=$direct O_DIRECT, auto-detected. 1 bypasses the OS page cache so we measure the device instead of RAM; 0 (buffered) is the fallback for filesystems/mounts that reject O_DIRECT (eg older OpenZFS, some NFS).
# --rw=randwrite means exactly what it looks like it means: we're going to do random write operations to our test files in the current working directory. Other options include seqread, seqwrite, randread, and randrw, all of which should hopefully be fairly self-explanatory. # --ramp_time=2s discard the first 2 seconds of results so we measure steady state, not the initial warm-up burst.
# --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk, and it also means a ton of extra overhead in the SATA, USB, SAS, SMB, or whatever other command channel lies between us and the disks, since a separate operation has to be commanded for each 4K of data. # --runtime=$runtime --time_based run for this many seconds, looping over the file(s) if we finish early.
# --size=1g our test file(s) will be 1GB in size apiece. (We're only creating one, see next argument.) # --group_reporting=1 aggregate all jobs into a single combined result line instead of one line per job.
# --numjobs=1 we're only creating a single file, and running a single process commanding operations within that file. If we wanted to simulate multiple parallel processes, we'd do, eg, --numjobs=16, which would create 16 separate test files of --size size, and 16 separate processes operating on them at the same time. # --end_fsync=1 after the timed run, flush everything to stable storage and count that time, so cached writes can't inflate the number.
# --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.) [ "$simple" != true ] && { echo ""; echo "4K Random Writes (bs=4k, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; }
# --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up.
# --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk.
[ "$simple" != true ] && { echo ""; echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; }
x=`sudo fio \ x=`sudo fio \
--name=fio-write-random-4k \ --name=fio-write-random-4k \
--directory=$path \ --directory=$path \
--ioengine=posixaio \ --ioengine=$engine \
--rw=randwrite \ --rw=randwrite \
--bs=4k \ --bs=4k \
--size=1g \ --size=256m \
--numjobs=1 \ --numjobs=4 \
--iodepth=1 \ --iodepth=64 \
--time_based --runtime=15 \ --direct=$direct \
--ramp_time=2s \
--time_based --runtime=$runtime \
--group_reporting=1 \
--end_fsync=1` --end_fsync=1`
print_result "Single 4K Random Writes" "$x" print_result "4K Random Writes" "$x"
# Cleanup my test files # Cleanup my test files
rm -rf $path/fio-write-random-4k* rm -rf $path/fio-write-random-4k*
@@ -158,10 +168,10 @@ function fio_write_parallel_random_64k {
# with lots of apps," think "busy fileserver with several people actively using it." # with lots of apps," think "busy fileserver with several people actively using it."
# You will see a lot more variation in speed as you watch this operation play # You will see a lot more variation in speed as you watch this operation play
# out on the console. For example, the 4K single process test we tried first # out on the console. Unlike the steady trickle you'd get from a queue-depth-1
# wrote a pretty consistent 11MiB/sec on my MacBook Air's internal drive—but # single-op latency test, this 16-process job can fluctuate wildly—eg between
# this 16-process job fluctuated between about 10MiB/sec and 300MiB/sec during # about 10MiB/sec and 300MiB/sec during the run—as the OS and SSD firmware
# the run, finishing with an average of 126MiB/sec. # catch good and bad luck aggregating writes.
# Most of the variation you're seeing here is due to the operating system and # Most of the variation you're seeing here is due to the operating system and
# SSD firmware sometimes being able to aggregate multiple writes. When it # SSD firmware sometimes being able to aggregate multiple writes. When it
@@ -171,17 +181,20 @@ function fio_write_parallel_random_64k {
# physical media stripe at a time—or a garbage collection or other maintenance # physical media stripe at a time—or a garbage collection or other maintenance
# operation at the SSD firmware level needs to run briefly in the background, # operation at the SSD firmware level needs to run briefly in the background,
# slowing things down. # slowing things down.
[ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"; } [ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (bs=64k, jobs=16, iodepth=16, engine=$engine, direct=$direct, ${runtime}s)"; }
x=`sudo fio \ x=`sudo fio \
--name=fio-write-random-64k \ --name=fio-write-random-64k \
--directory=$path \ --directory=$path \
--ioengine=posixaio \ --ioengine=$engine \
--rw=randwrite \ --rw=randwrite \
--bs=64k \ --bs=64k \
--size=64m \ --size=64m \
--numjobs=16 \ --numjobs=16 \
--iodepth=16 \ --iodepth=16 \
--time_based --runtime=15 \ --direct=$direct \
--ramp_time=2s \
--time_based --runtime=$runtime \
--group_reporting=1 \
--end_fsync=1` --end_fsync=1`
print_result "Parallel 64K Random Writes" "$x" print_result "Parallel 64K Random Writes" "$x"
@@ -190,7 +203,7 @@ function fio_write_parallel_random_64k {
} }
function fio_write_single_sequential_1m { function fio_write_single_sequential_1m {
# Single 1M Random Writes # Single 1M Sequential Writes
# This is pretty close to the best-case scenario for a real-world system # This is pretty close to the best-case scenario for a real-world system
# doing real-world things. No, it's not quite as fast as a single, truly # doing real-world things. No, it's not quite as fast as a single, truly
@@ -228,38 +241,41 @@ function fio_write_single_sequential_1m {
# here is that you understand both what you want to test, and how to test # here is that you understand both what you want to test, and how to test
# it accurately. # it accurately.
[ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; } [ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (bs=1m, jobs=1, iodepth=16, engine=$engine, direct=$direct, ${runtime}s)"; }
x=`sudo fio \ x=`sudo fio \
--name=fio-write-random-1m \ --name=fio-write-seq-1m \
--directory=$path \ --directory=$path \
--ioengine=posixaio \ --ioengine=$engine \
--rw=write \ --rw=write \
--bs=1m \ --bs=1m \
--size=1g \ --size=1g \
--numjobs=1 \ --numjobs=1 \
--iodepth=1 \ --iodepth=16 \
--time_based --runtime=15 \ --direct=$direct \
--ramp_time=2s \
--time_based --runtime=$runtime \
--group_reporting=1 \
--end_fsync=1` --end_fsync=1`
print_result "Single 1M Sequential Writes" "$x" print_result "Single 1M Sequential Writes" "$x"
# Cleanup my test files # Cleanup my test files
rm -rf $path/fio-write-random-1m* rm -rf $path/fio-write-seq-1m*
} }
function fio_read_sequential_1m { function fio_read_sequential_1m {
# Sequential Parallel Reads # Sequential Parallel Reads
[ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads"; } [ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads (bs=1m, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; }
x=`sudo fio \ x=`sudo fio \
--name=fio-read-sequential-1m \ --name=fio-read-sequential-1m \
--directory=$path \ --directory=$path \
--ioengine=posixaio \ --ioengine=$engine \
--bs=1M \ --bs=1M \
--numjobs=4 \ --numjobs=4 \
--size=256M \ --size=256M \
--time_based --runtime=30s \ --time_based --runtime=$runtime \
--ramp_time=2s \ --ramp_time=2s \
--direct=1 \ --direct=$direct \
--verify=0 \ --verify=0 \
--iodepth=64 \ --iodepth=64 \
--rw=read \ --rw=read \
@@ -271,30 +287,91 @@ function fio_read_sequential_1m {
} }
function fio_read_random_4k { function fio_read_random_4k {
# Random 4k Reads # Random 4k Reads (parallel, deep queue)
[ "$simple" != true ] && { echo ""; echo "Random 4k Reads"; } # 4K random reads are the highest-IOPS thing a drive does, but a single
# submitting thread goes CPU-bound long before an NVMe runs out of capacity,
# so it under-reports peak IOPS. We spread the work over 4 jobs x iodepth=64
# (256 ops in flight, like the 4K write test) to saturate fast drives, while
# still reporting the honest (low) number on a spinning HDD. size=256m per
# job keeps the total footprint at ~1GB across the 4 jobs.
[ "$simple" != true ] && { echo ""; echo "Random 4k Reads (bs=4k, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; }
x=`sudo fio \ x=`sudo fio \
--name=fio-read-random-4k \ --name=fio-read-random-4k \
--directory=$path \ --directory=$path \
--ioengine=posixaio \ --ioengine=$engine \
--rw=randread \ --rw=randread \
--bs=4k \ --bs=4k \
--size=1g \ --size=256m \
--time_based --runtime=30s \ --numjobs=4 \
--time_based --runtime=$runtime \
--ramp_time=2s \ --ramp_time=2s \
--direct=1 \ --direct=$direct \
--verify=0 \ --verify=0 \
--iodepth=256 \ --iodepth=64 \
--rw=read \
--group_reporting=1 \ --group_reporting=1 \
--iodepth_batch_submit=256 \ --iodepth_batch_submit=64 \
--iodepth_batch_complete_max=256` --iodepth_batch_complete_max=64`
print_result "Random 4k Reads" "$x" print_result "Random 4k Reads" "$x"
rm -rf $path/fio-read-random-4k* rm -rf $path/fio-read-random-4k*
} }
function fio_probe {
# Run a tiny throwaway fio job to see if a given (engine, direct) combo
# actually works on this path/filesystem. Returns 0 on success. This is how
# we stay accurate AND portable across ext4, xfs, btrfs, ZFS, NFS, etc.
# without the caller needing to know what each filesystem supports.
local eng="$1" dir="$2"
sudo fio --name=probe \
--directory="$path" \
--ioengine="$eng" \
--rw=write --bs=4k --size=1m \
--direct="$dir" \
--time_based --runtime=1 \
> /dev/null 2>&1
local rc=$?
rm -rf "$path"/probe* 2>/dev/null
return $rc
}
function detect_io_settings {
# Pick the fastest available IO engine unless the user forced one with
# --engine=. io_uring is the modern, lowest-overhead Linux engine; libaio
# is older (and only truly async with O_DIRECT); posixaio/sync are the
# portable fallbacks (eg some NFS mounts).
if [ -z "$engine" ]; then
for eng in io_uring libaio posixaio sync; do
if fio_probe "$eng" 0; then engine="$eng"; break; fi
done
[ -z "$engine" ] && engine="sync"
fi
# Decide O_DIRECT unless forced with --direct/--buffered. O_DIRECT bypasses
# the OS page cache so we measure the device instead of RAM. Not every
# filesystem supports it (eg older OpenZFS < 2.3, some NFS mounts), so we
# probe and fall back to buffered rather than erroring out.
if [ -z "$direct" ]; then
if fio_probe "$engine" 1; then direct=1; else direct=0; fi
fi
if [ "$simple" != true ]; then
echo ""
echo "IO engine : $engine"
if [ "$direct" == 1 ]; then
echo "O_DIRECT : enabled (bypassing page cache -- measuring the device)"
else
echo "O_DIRECT : DISABLED (buffered) -- this filesystem/mount rejected"
echo " O_DIRECT, so results (especially reads) may reflect the"
echo " RAM cache rather than the underlying device. Pass"
echo " --direct to force it if you believe it should work."
fi
fi
}
function fio_speedtest { function fio_speedtest {
# Auto-detect the best engine and whether O_DIRECT works on this path.
detect_io_settings
# Write tests # Write tests
fio_write_single_random_4k fio_write_single_random_4k
fio_write_parallel_random_64k fio_write_parallel_random_64k
@@ -359,6 +436,15 @@ function usage {
echo " Add --simple (FIO only) for a compact, aligned summary of MB/s values" echo " Add --simple (FIO only) for a compact, aligned summary of MB/s values"
echo " ./speedtest-hd . --simple" echo " ./speedtest-hd . --simple"
echo " ./speedtest-hd . --fio --simple" echo " ./speedtest-hd . --fio --simple"
echo ""
echo " FIO tuning flags (all auto-detected by default, override as needed):"
echo " --engine=io_uring|libaio|posixaio|sync IO engine (default: auto)"
echo " --direct Force O_DIRECT (bypass page cache)"
echo " --buffered Force buffered IO (eg if O_DIRECT is unsupported)"
echo " --runtime=SEC Seconds per test (default: 30)"
echo " Examples:"
echo " ./speedtest-hd /mnt/nvmepool --engine=io_uring --runtime=60"
echo " ./speedtest-hd /mnt/nfsshare --buffered"
exit 0 exit 0
} }