From 4417a668bcf47d4294b65c6f92cc785bd7a913d3 Mon Sep 17 00:00:00 2001 From: Matthew Reschke Date: Sun, 21 Jun 2026 17:34:17 -0600 Subject: [PATCH] Claude pass, based mods to queuedepth and params --- speedtest-hd.sh | 212 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 149 insertions(+), 63 deletions(-) diff --git a/speedtest-hd.sh b/speedtest-hd.sh index f954189..fa85f83 100755 --- a/speedtest-hd.sh +++ b/speedtest-hd.sh @@ -11,11 +11,19 @@ path="$1" option="" simple=false +engine="" # IO engine; empty = auto-detect (io_uring > libaio > posixaio > sync) +direct="" # O_DIRECT; empty = auto-detect; 1 = bypass page cache, 0 = buffered +runtime=30 # seconds per test (longer = more accurate, exposes SLC cache exhaustion) for arg in "${@:2}"; do case "$arg" in --simple) simple=true ;; --dd) option="--dd" ;; --fio) option="--fio" ;; + --buffered) direct=0 ;; + --direct) direct=1 ;; + --engine=*) engine="${arg#*=}" ;; + --direct=*) direct="${arg#*=}" ;; + --runtime=*) runtime="${arg#*=}" ;; esac done @@ -86,50 +94,52 @@ function print_result { } function fio_write_single_random_4k { - # Single 4k Random Writes + # 4K Random Writes (parallel, deep queue) - # This is a single process doing random 4K writes. This is where the pain - # really, really lives; it's basically the worst possible thing you can ask a - # disk to do. Where this happens most frequently in real life: copying home - # directories and dotfiles, manipulating email stuff, some database operations, - # source code trees. + # Random 4K writes are the worst thing you can ask a disk to do. Where this + # happens most in real life: copying home directories and dotfiles, email + # stores, some database operations, source code trees. - # When I ran this test against the high-performance SSDs in my Ubuntu - # workstation, they pushed 127MiB/sec. The server just beneath it in the rack - # only managed 33MiB/sec on its "high-performance" 7200RPM rust disks... but - # even then, the vast majority of that speed is because the data is being - # written asynchronously, allowing the operating system to batch it up into - # larger, more efficient write operations. + # IMPORTANT: the original version of this test used numjobs=1 + iodepth=1. + # That measures single-op LATENCY (the absolute worst case): the OS issues + # one 4K write, waits for it to be acknowledged, then issues the next. On + # NVMe that lands around ~3000 IOPS / ~12MB/sec -- which looks alarmingly + # slow but is NOT a bug. NVMe's entire advantage is deep command queues and + # parallelism, so a queue depth of 1 deliberately measures the one scenario + # NVMe is bad at. To measure the drive's actual capability we now drive it + # with multiple jobs and a deep queue (4 jobs x iodepth=64), keeping + # hundreds of operations in flight at once. Expect this to jump into the + # hundreds of MB/sec (often 1GB/sec+) on a healthy NVMe pool. - # If we add the argument --fsync=1, forcing the operating system to perform - # synchronous writes (calling fsync after each block of data is written) the - # picture gets much more grim: 2.6MiB/sec on the high-performance SSDs but - # only 184KiB/sec on the "high-performance" rust. The SSDs were about four - # times faster than the rust when data was written asynchronously but a - # whopping fourteen times faster when - - # --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the working directory you're currently in. - # --ioengine=posixaio sets the mode fio interacts with the filesystem. POSIX is a standard Windows, Macs, Linux, and BSD all understand, so it's great for portability—although inside fio itself, Windows users need to invoke --ioengine=windowsaio, not --ioengine=posixaio, unfortunately. AIO stands for Asynchronous Input Output and means that we can queue up multiple operations to be completed in whatever order the OS decides to complete them. (In this particular example, later arguments effectively nullify this.) - # --rw=randwrite means exactly what it looks like it means: we're going to do random write operations to our test files in the current working directory. Other options include seqread, seqwrite, randread, and randrw, all of which should hopefully be fairly self-explanatory. - # --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk, and it also means a ton of extra overhead in the SATA, USB, SAS, SMB, or whatever other command channel lies between us and the disks, since a separate operation has to be commanded for each 4K of data. - # --size=1g our test file(s) will be 1GB in size apiece. (We're only creating one, see next argument.) - # --numjobs=1 we're only creating a single file, and running a single process commanding operations within that file. If we wanted to simulate multiple parallel processes, we'd do, eg, --numjobs=16, which would create 16 separate test files of --size size, and 16 separate processes operating on them at the same time. - # --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.) - # --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up. - # --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk. - [ "$simple" != true ] && { echo ""; echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; } + # --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the directory you point it at. + # --directory= where the test files are created; this is the path/mount you are benchmarking. + # --ioengine=$engine the engine fio uses to talk to the kernel, auto-detected by detect_io_settings(). On modern Linux this is io_uring (lowest overhead); it falls back to libaio, then posixaio, then sync. NOTE: libaio is only truly asynchronous with --direct=1; with buffered IO it silently degrades to synchronous, which is exactly why io_uring is preferred. + # --rw=randwrite random write operations. Other options include read, write (sequential), randread, and randrw. + # --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk and adds a ton of command-channel overhead, since a separate operation has to be commanded for each 4K of data. + # --size=256m each job's test file is 256MB. With --numjobs=4 that's 4 files totaling ~1GB. + # --numjobs=4 run 4 parallel processes, each with its own test file, to feed the drive from multiple threads at once (single-threaded cannot saturate NVMe). + # --iodepth=64 how many commands each job keeps stacked in the queue at once. Deep queues are how NVMe reaches its real IOPS; 4 jobs x 64 = up to 256 ops in flight. + # --direct=$direct O_DIRECT, auto-detected. 1 bypasses the OS page cache so we measure the device instead of RAM; 0 (buffered) is the fallback for filesystems/mounts that reject O_DIRECT (eg older OpenZFS, some NFS). + # --ramp_time=2s discard the first 2 seconds of results so we measure steady state, not the initial warm-up burst. + # --runtime=$runtime --time_based run for this many seconds, looping over the file(s) if we finish early. + # --group_reporting=1 aggregate all jobs into a single combined result line instead of one line per job. + # --end_fsync=1 after the timed run, flush everything to stable storage and count that time, so cached writes can't inflate the number. + [ "$simple" != true ] && { echo ""; echo "4K Random Writes (bs=4k, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; } x=`sudo fio \ --name=fio-write-random-4k \ --directory=$path \ - --ioengine=posixaio \ + --ioengine=$engine \ --rw=randwrite \ --bs=4k \ - --size=1g \ - --numjobs=1 \ - --iodepth=1 \ - --time_based --runtime=15 \ + --size=256m \ + --numjobs=4 \ + --iodepth=64 \ + --direct=$direct \ + --ramp_time=2s \ + --time_based --runtime=$runtime \ + --group_reporting=1 \ --end_fsync=1` - print_result "Single 4K Random Writes" "$x" + print_result "4K Random Writes" "$x" # Cleanup my test files rm -rf $path/fio-write-random-4k* @@ -158,10 +168,10 @@ function fio_write_parallel_random_64k { # with lots of apps," think "busy fileserver with several people actively using it." # You will see a lot more variation in speed as you watch this operation play - # out on the console. For example, the 4K single process test we tried first - # wrote a pretty consistent 11MiB/sec on my MacBook Air's internal drive—but - # this 16-process job fluctuated between about 10MiB/sec and 300MiB/sec during - # the run, finishing with an average of 126MiB/sec. + # out on the console. Unlike the steady trickle you'd get from a queue-depth-1 + # single-op latency test, this 16-process job can fluctuate wildly—eg between + # about 10MiB/sec and 300MiB/sec during the run—as the OS and SSD firmware + # catch good and bad luck aggregating writes. # Most of the variation you're seeing here is due to the operating system and # SSD firmware sometimes being able to aggregate multiple writes. When it @@ -171,17 +181,20 @@ function fio_write_parallel_random_64k { # physical media stripe at a time—or a garbage collection or other maintenance # operation at the SSD firmware level needs to run briefly in the background, # slowing things down. - [ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"; } + [ "$simple" != true ] && { echo ""; echo "Parallel 64K Random Writes (bs=64k, jobs=16, iodepth=16, engine=$engine, direct=$direct, ${runtime}s)"; } x=`sudo fio \ --name=fio-write-random-64k \ --directory=$path \ - --ioengine=posixaio \ + --ioengine=$engine \ --rw=randwrite \ --bs=64k \ --size=64m \ --numjobs=16 \ --iodepth=16 \ - --time_based --runtime=15 \ + --direct=$direct \ + --ramp_time=2s \ + --time_based --runtime=$runtime \ + --group_reporting=1 \ --end_fsync=1` print_result "Parallel 64K Random Writes" "$x" @@ -190,7 +203,7 @@ function fio_write_parallel_random_64k { } function fio_write_single_sequential_1m { - # Single 1M Random Writes + # Single 1M Sequential Writes # This is pretty close to the best-case scenario for a real-world system # doing real-world things. No, it's not quite as fast as a single, truly @@ -228,38 +241,41 @@ function fio_write_single_sequential_1m { # here is that you understand both what you want to test, and how to test # it accurately. - [ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"; } + [ "$simple" != true ] && { echo ""; echo "Single 1M Sequential Writes (bs=1m, jobs=1, iodepth=16, engine=$engine, direct=$direct, ${runtime}s)"; } x=`sudo fio \ - --name=fio-write-random-1m \ + --name=fio-write-seq-1m \ --directory=$path \ - --ioengine=posixaio \ + --ioengine=$engine \ --rw=write \ --bs=1m \ --size=1g \ --numjobs=1 \ - --iodepth=1 \ - --time_based --runtime=15 \ + --iodepth=16 \ + --direct=$direct \ + --ramp_time=2s \ + --time_based --runtime=$runtime \ + --group_reporting=1 \ --end_fsync=1` print_result "Single 1M Sequential Writes" "$x" # Cleanup my test files - rm -rf $path/fio-write-random-1m* + rm -rf $path/fio-write-seq-1m* } function fio_read_sequential_1m { # Sequential Parallel Reads - [ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads"; } + [ "$simple" != true ] && { echo ""; echo "Sequential 4x 1M Reads (bs=1m, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; } x=`sudo fio \ --name=fio-read-sequential-1m \ --directory=$path \ - --ioengine=posixaio \ + --ioengine=$engine \ --bs=1M \ --numjobs=4 \ --size=256M \ - --time_based --runtime=30s \ + --time_based --runtime=$runtime \ --ramp_time=2s \ - --direct=1 \ + --direct=$direct \ --verify=0 \ --iodepth=64 \ --rw=read \ @@ -271,30 +287,91 @@ function fio_read_sequential_1m { } function fio_read_random_4k { - # Random 4k Reads + # Random 4k Reads (parallel, deep queue) - [ "$simple" != true ] && { echo ""; echo "Random 4k Reads"; } + # 4K random reads are the highest-IOPS thing a drive does, but a single + # submitting thread goes CPU-bound long before an NVMe runs out of capacity, + # so it under-reports peak IOPS. We spread the work over 4 jobs x iodepth=64 + # (256 ops in flight, like the 4K write test) to saturate fast drives, while + # still reporting the honest (low) number on a spinning HDD. size=256m per + # job keeps the total footprint at ~1GB across the 4 jobs. + [ "$simple" != true ] && { echo ""; echo "Random 4k Reads (bs=4k, jobs=4, iodepth=64, engine=$engine, direct=$direct, ${runtime}s)"; } x=`sudo fio \ --name=fio-read-random-4k \ --directory=$path \ - --ioengine=posixaio \ + --ioengine=$engine \ --rw=randread \ --bs=4k \ - --size=1g \ - --time_based --runtime=30s \ + --size=256m \ + --numjobs=4 \ + --time_based --runtime=$runtime \ --ramp_time=2s \ - --direct=1 \ + --direct=$direct \ --verify=0 \ - --iodepth=256 \ - --rw=read \ + --iodepth=64 \ --group_reporting=1 \ - --iodepth_batch_submit=256 \ - --iodepth_batch_complete_max=256` + --iodepth_batch_submit=64 \ + --iodepth_batch_complete_max=64` print_result "Random 4k Reads" "$x" rm -rf $path/fio-read-random-4k* } +function fio_probe { + # Run a tiny throwaway fio job to see if a given (engine, direct) combo + # actually works on this path/filesystem. Returns 0 on success. This is how + # we stay accurate AND portable across ext4, xfs, btrfs, ZFS, NFS, etc. + # without the caller needing to know what each filesystem supports. + local eng="$1" dir="$2" + sudo fio --name=probe \ + --directory="$path" \ + --ioengine="$eng" \ + --rw=write --bs=4k --size=1m \ + --direct="$dir" \ + --time_based --runtime=1 \ + > /dev/null 2>&1 + local rc=$? + rm -rf "$path"/probe* 2>/dev/null + return $rc +} + +function detect_io_settings { + # Pick the fastest available IO engine unless the user forced one with + # --engine=. io_uring is the modern, lowest-overhead Linux engine; libaio + # is older (and only truly async with O_DIRECT); posixaio/sync are the + # portable fallbacks (eg some NFS mounts). + if [ -z "$engine" ]; then + for eng in io_uring libaio posixaio sync; do + if fio_probe "$eng" 0; then engine="$eng"; break; fi + done + [ -z "$engine" ] && engine="sync" + fi + + # Decide O_DIRECT unless forced with --direct/--buffered. O_DIRECT bypasses + # the OS page cache so we measure the device instead of RAM. Not every + # filesystem supports it (eg older OpenZFS < 2.3, some NFS mounts), so we + # probe and fall back to buffered rather than erroring out. + if [ -z "$direct" ]; then + if fio_probe "$engine" 1; then direct=1; else direct=0; fi + fi + + if [ "$simple" != true ]; then + echo "" + echo "IO engine : $engine" + if [ "$direct" == 1 ]; then + echo "O_DIRECT : enabled (bypassing page cache -- measuring the device)" + else + echo "O_DIRECT : DISABLED (buffered) -- this filesystem/mount rejected" + echo " O_DIRECT, so results (especially reads) may reflect the" + echo " RAM cache rather than the underlying device. Pass" + echo " --direct to force it if you believe it should work." + fi + fi +} + function fio_speedtest { + # Auto-detect the best engine and whether O_DIRECT works on this path. + detect_io_settings + # Write tests fio_write_single_random_4k fio_write_parallel_random_64k @@ -359,6 +436,15 @@ function usage { echo " Add --simple (FIO only) for a compact, aligned summary of MB/s values" echo " ./speedtest-hd . --simple" echo " ./speedtest-hd . --fio --simple" + echo "" + echo " FIO tuning flags (all auto-detected by default, override as needed):" + echo " --engine=io_uring|libaio|posixaio|sync IO engine (default: auto)" + echo " --direct Force O_DIRECT (bypass page cache)" + echo " --buffered Force buffered IO (eg if O_DIRECT is unsupported)" + echo " --runtime=SEC Seconds per test (default: 30)" + echo " Examples:" + echo " ./speedtest-hd /mnt/nvmepool --engine=io_uring --runtime=60" + echo " ./speedtest-hd /mnt/nfsshare --buffered" exit 0 }