Many updates over time

README.md

@@ -70,6 +70,53 @@ nocows = 1
 ```
 
 
+# OS Distro and Version
+
+How to determine `ansible_distribution` for an exact host?
+
+`ansible -i vlab/env.yml 'linprox' -m ansible.builtin.setup -a "filter=ansible_distribution*"`
+
+```
+ansible_os_family
+    Will say Debian for Debian AND Ubuntu
+    Good to call files that work for BOTH Debian and Ubuntu
+    Debian
+    Archlinux
+
+ansible_lsb.id
+    This should be SAME as ansible_distribution
+    But it relies on lsb-releases package being installed
+    Best to use ansible_distribution as a standard
+    NOTE: For Manjaro ansible_distribution=Manjaro
+    but ansible_lsb.id=ManjaroLinux
+
+ansible_distribution
+    Debian
+    Ubuntu
+    Fedora
+    RedHat
+    Archlinux
+    Manjaro
+
+ansible_distribution_major_version
+    8
+    9
+    10
+    11
+    12
+
+    18
+    20
+    22
+    24
+
+ansible_distribution_version
+    22.04
+    22.10
+    24.04
+    24.10
+```
+
 
 # Snippets
 
@@ -83,10 +130,10 @@ when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "16.0
 when: ansible_os_family == "Archlinux" and ansible_lsb.id == "ManjaroLinux"
 
 # Copy profiles to /etc/profile.d/
-- include_tasks: ../../../functions/copy_etc-profile.d.yml
+- include_tasks: ../../shared/functions/copy_etc-profile.d.yml
 
 # Copy scripts to /usr/local/bin
-- include_tasks: ../../../functions/copy_usr-local-bin.yml
+- include_tasks: ../../shared/functions/copy_usr-local-bin.yml
 
 # Copy globs
 - name: Copying globs

app/gitlab-runner/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Add GitLab Runner repository script
+  command: curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash
+  args:
+    creates: /etc/apt/sources.list.d/runner_gitlab-runner.list
+
+- name: Install GitLab Runner and Helper packages
+  ansible.builtin.apt:
+    name: gitlab-runner
+    state: present
+    update_cache: yes

code/poetry/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 # Copy profiles to /etc/profile.d/
-- include_tasks: ../../../functions/copy_etc-profile.d.yml
+- include_tasks: ../../shared/functions/copy_etc-profile.d.yml
 
 # Create directory
 - name: Create /etc/poetry directory

code/pyenv/tasks/install-pyenv.yml

@@ -38,6 +38,24 @@
       - libffi-dev
   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
 
+# Debian 12
+- name: Installing build-essential and pyenv dependencies
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - build-essential
+      - libreadline-dev
+      - libncursesw5-dev
+      - libssl-dev
+      - libsqlite3-dev
+      - tk-dev
+      - libgdbm-dev
+      - libc6-dev
+      - libbz2-dev
+      - libffi-dev
+  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "12"
+
 # Install pyenv using git clone
 - name: Installing pyenv using git clone
   git:
@@ -52,3 +70,10 @@
 - file: 'path=/usr/local/lib/pyenv owner={{ superuser }} group={{ supergroup }} state=directory recurse=yes mode=2775'
 - command: find /usr/local/lib/pyenv -type d -exec chmod 2775 {} \;
 - command: setfacl -R -d -m user::rwx,group::rwx /usr/local/lib/pyenv
+
+# Adjust git to allow alternate perms on /usr/local/lib/pyenv
+# If you don't do this, and run playbook again it errors on
+# FAILED! => {"changed": false, "msg": "Failed to set a new url https://github.com/pyenv/pyenv.git for origin:  fatal: detected dubious ownership in repository at '/usr/local/lib/pyenv'\nTo add an exception for this directory, call:\n\n\tgit config --global --add safe.directory /usr/local/lib/pyenv\n"}
+- name: Adjust git to allow alternate perms on /usr/local/lib/pyenv
+  ansible.builtin.shell:
+    cmd: git config --global --add safe.directory /usr/local/lib/pyenv

code/pyenv/tasks/main.yml

@@ -1,9 +1,9 @@
 ---
 # Copy profiles to /etc/profile.d/
-- include_tasks: ../../../functions/copy_etc-profile.d.yml
+- include_tasks: ../../shared/functions/copy_etc-profile.d.yml
 
 # Copy scripts to /usr/local/bin
-- include_tasks: ../../../functions/copy_usr-local-bin.yml
+- include_tasks: ../../shared/functions/copy_usr-local-bin.yml
 
 # Install pyenv
 - include_tasks: install-pyenv.yml

file/autofs/README.md

@@ -0,0 +1,14 @@
+# Usage
+
+In a playbook
+
+```yml
+    # Configure autofs
+    - role: shared/roles/file/autofs
+      autofs_mounts:
+        - src: linvault1:/mnt/store/users/mreschke/Pictures
+          dest: /mnt/nas/users/mreschke/Pictures
+        - src: linvault1:/mnt/store/users/allison/Pictures
+          dest: /mnt/nas/users/allison/Pictures
+
+```

file/autofs/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: reload-autofs
+  service:
+    name: autofs
+    state: reloaded

file/autofs/tasks/Debian/Debian_13.yml

@@ -0,0 +1,38 @@
+---
+
+# Install autofs
+- name: Installing AutoFS on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - autofs
+
+# Create extra.autofs
+- name: Creating AutoFS /etc/auto.master.d/extra.autofs
+  template:
+    src: extra.autofs.j2
+    dest: /etc/auto.master.d/extra.autofs
+    owner: root
+    group: root
+    mode: '0644'
+
+# Create the dest directories
+- name: Creating AutoFS destination directories
+  file:
+    path: "{{ item.dest }}"
+    state: directory
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    mode: '0755'
+  loop: "{{ autofs_mounts }}"
+
+# Create each servers /etc/auto.master.d/server.autofs file
+- name: Creating AutoFS /etc/auto.master.d/server.autofs for each server in autofs_mounts
+  template:
+    src: mounts.autofs.j2
+    dest: /etc/auto.master.d/mounts.autofs
+    owner: root
+    group: root
+    mode: '0644'
+  notify: reload-autofs

file/autofs/tasks/Debian/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Dynamically include proper file based on OS and Version
+- name: "Configuring AutoFS on a {{ ansible_distribution }} {{ ansible_distribution_major_version }} System"
+  include_tasks: "{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml"

file/autofs/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+
+# OS Specific main.yml
+- name: "Configuring AutoFS on a {{ ansible_distribution }} System"
+  include_tasks: "{{ ansible_distribution }}/main.yml"

file/autofs/templates/extra.autofs.j2

@@ -0,0 +1 @@
+/-      /etc/auto.master.d/mounts.autofs

file/autofs/templates/mounts.autofs.j2

@@ -0,0 +1,4 @@
+{% for mount in autofs_mounts %}
+{{mount.dest}} -fstype=nfs,rw,soft,intr {{mount.src}}
+{% endfor %}
+

file/syncthing/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# Install Syncthing
+- name: Installing Syncthing
+  apt:
+    update_cache: yes
+    state: present
+    name: syncthing

functions/copy_etc-profile.d.yml

@@ -1,24 +0,0 @@
-# Copy profiles to /etc/profile.d/
-# Templates messes up a lot of back scripts (errors on odd chars)
-# So copy all files/bin/* using COPY
-- name: Copying profiles to /etc/profile.d
-  copy:
-    src: "{{ item }}"
-    dest: /etc/profile.d/
-    owner: root
-    group: root
-    mode: '0755'
-  with_fileglob:
-    - files/profile.d/*
-
-# Template and copy profiles to /etc/profile.d/
-# Careful here, as some script can mess up the template system (jinja2)
-- name: Copy templated profiles to /etc/profile.d/
-  template:
-    src: "{{ item }}"
-    dest: /etc/profile.d/
-    owner: root
-    group: root
-    mode: '0755'
-  with_fileglob:
-    - templates/profile.d/*

functions/copy_usr-local-bin.yml

@@ -1,24 +0,0 @@
-# Copy scripts to /usr/local/bin
-# Templates messes up a lot of back scripts (errors on odd chars)
-# So copy all files/bin/* using COPY
-- name: Copying scripts to /usr/local/bin
-  copy:
-    src: "{{ item }}"
-    dest: /usr/local/bin/
-    owner: '{{ superuser }}'
-    group: '{{ supergroup }}'
-    mode: '0775'
-  with_fileglob:
-    - files/bin/*
-
-# Template and copy scripts to /usr/local/bin
-# Careful here, as some script can mess up the template system (jinja2)
-- name: Copy templated scripts to /usr/local/bin
-  template:
-    src: "{{ item }}"
-    dest: /usr/local/bin/
-    owner: '{{ superuser }}'
-    group: '{{ supergroup }}'
-    mode: '0775'
-  with_fileglob:
-    - templates/bin/*

server/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+type: 'virtual'
+modify_apt_sources: true

server/files/Debian/12/sources.list

@@ -0,0 +1,8 @@
+# Debian 12 Bookworm Software
+deb http://deb.debian.org/debian bookworm main contrib non-free non-free-firmware
+
+# Debian 12 Bookworm Updates
+deb http://deb.debian.org/debian bookworm-updates main contrib non-free non-free-firmware
+
+# Debian 12 Bookworm Security
+deb http://security.debian.org/debian-security bookworm-security main contrib non-free non-free-firmware

server/files/Debian/13/sources.list

@@ -0,0 +1,8 @@
+# Debian 13 Trixie Software
+deb http://deb.debian.org/debian/ trixie main contrib non-free non-free-firmware
+
+# Debian 13 Trixie Updates
+deb http://deb.debian.org/debian/ trixie-updates main contrib non-free non-free-firmware
+
+# Debian 13 Trixie Security
+deb http://security.debian.org/debian-security trixie-security main contrib non-free non-free-firmware

server/files/bin/speedtest-hd

@@ -1,11 +1,296 @@
 #!/usr/bin/env bash
 
+# Robust HD/SDD/NVMe performance CLI utility
+# Utilizing FIO for sequential/random writes/writes
+# Dependencies: fio (apt install fio)
+# See: https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance
+# See: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+# mReschke 2024-01-18
+
+# CLI Parameters
+path="$1"
+option="$2"
+
+# Main application flow
+function main {
+
+    # Show usage if no params
+    if [ ! "$path" ]; then
+        usage
+    fi
+
+    # Understand . path
+    if [ "$path" == '.' ]; then
+        path=$(pwd)
+    fi
+
+    # Check if path exists
+    if [ ! -e "$path" ]; then
+        echo "Path $path does not exist"
+        exit 1
+    fi
+
+    # Must type y or n THEN press enter (which I like better)
+    echo "NOTICE: 1GB free space on '$path' is required to perform the benchmark."
+    echo -n "Are you ready to start a robust IO benchmark against '$path' ?"; read answer
+    if [ "$answer" != "${answer#[Yy]}" ]; then
+        echo "Great! Starting benchmark now!";
+    else
+        echo "Ok, cancelled!"
+        exit 0
+    fi
+
+    # Use dd of fio based on param or defaults
+    if [ "$option" == "--dd" ]; then
+        dd_speedtest
+    elif [ "$option" == "--fio" ]; then
+        fio_speedtest
+    elif [ "$option" == "" ]; then
+        # If fio is installed, use it, else use dd
+        echo ""
+        if ! command -v fio &> /dev/null; then
+            dd_speedtest
+        else
+            fio_speedtest
+        fi
+    fi
+}
+
+function fio_write_single_random_4k {
+    # Single 4k Random Writes
+
+    # This is a single process doing random 4K writes. This is where the pain
+    # really, really lives; it's basically the worst possible thing you can ask a
+    # disk to do. Where this happens most frequently in real life: copying home
+    # directories and dotfiles, manipulating email stuff, some database operations,
+    # source code trees.
+
+    # When I ran this test against the high-performance SSDs in my Ubuntu
+    # workstation, they pushed 127MiB/sec. The server just beneath it in the rack
+    # only managed 33MiB/sec on its "high-performance" 7200RPM rust disks... but
+    # even then, the vast majority of that speed is because the data is being
+    # written asynchronously, allowing the operating system to batch it up into
+    # larger, more efficient write operations.
+
+    # If we add the argument --fsync=1, forcing the operating system to perform
+    # synchronous writes (calling fsync after each block of data is written) the
+    # picture gets much more grim: 2.6MiB/sec on the high-performance SSDs but
+    # only 184KiB/sec on the "high-performance" rust. The SSDs were about four
+    # times faster than the rust when data was written asynchronously but a
+    # whopping fourteen times faster when 
+
+    # --name= is a required argument, but it's basically human-friendly fluff—fio will create files based on that name to test with, inside the working directory you're currently in.    
+    # --ioengine=posixaio sets the mode fio interacts with the filesystem. POSIX is a standard Windows, Macs, Linux, and BSD all understand, so it's great for portability—although inside fio itself, Windows users need to invoke --ioengine=windowsaio, not --ioengine=posixaio, unfortunately. AIO stands for Asynchronous Input Output and means that we can queue up multiple operations to be completed in whatever order the OS decides to complete them. (In this particular example, later arguments effectively nullify this.)
+    # --rw=randwrite means exactly what it looks like it means: we're going to do random write operations to our test files in the current working directory. Other options include seqread, seqwrite, randread, and randrw, all of which should hopefully be fairly self-explanatory.
+    # --bs=4k blocksize 4K. These are very small individual operations. This is where the pain lives; it's hard on the disk, and it also means a ton of extra overhead in the SATA, USB, SAS, SMB, or whatever other command channel lies between us and the disks, since a separate operation has to be commanded for each 4K of data.
+    # --size=1g our test file(s) will be 1GB in size apiece. (We're only creating one, see next argument.)
+    # --numjobs=1 we're only creating a single file, and running a single process commanding operations within that file. If we wanted to simulate multiple parallel processes, we'd do, eg, --numjobs=16, which would create 16 separate test files of --size size, and 16 separate processes operating on them at the same time.
+    # --iodepth=1 this is how deep we're willing to try to stack commands in the OS's queue. Since we set this to 1, this is effectively pretty much the same thing as the sync IO engine—we're only asking for a single operation at a time, and the OS has to acknowledge receipt of every operation we ask for before we can ask for another. (It does not have to satisfy the request itself before we ask it to do more operations, it just has to acknowledge that we actually asked for it.)
+    # --runtime=15 --time_based Run and even if we complete sooner, just start over again and keep going until 60 seconds is up.
+    # --end_fsync=1 After all operations have been queued, keep the timer going until the OS reports that the very last one of them has been successfully completed—ie, actually written to disk.
+    echo ""
+    echo "Single 4K Random Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
+    x=`sudo fio \
+        --name=fio-write-random-4k \
+        --directory=$path \
+        --ioengine=posixaio \
+        --rw=randwrite \
+        --bs=4k \
+        --size=1g \
+        --numjobs=1 \
+        --iodepth=1 \
+        --time_based --runtime=15 \
+        --end_fsync=1`
+    echo "  - $x " | /usr/bin/grep -A1 'Run status group' | tail -n1
+    
+    # Cleanup my test files
+    rm -rf $path/fio-write-random-4k*
+}
+
+function fio_write_parallel_random_64k {
+    # Parallel 64k Random Writes
+
+    # This time, we're creating 16 separate 64MB files (still totaling 1GB, when
+    # all put together) and we're issuing 64KB blocksized random write operations.
+    # We're doing it with sixteen separate processes running in parallel, and
+    # we're queuing up to 16 simultaneous asynchronous ops before we pause and wait
+    # for the OS to start acknowledging their receipt.
+
+    # This is a pretty decent approximation of a significantly busy system. It's
+    # not doing any one particularly nasty thing—like running a database engine or
+    # copying tons of dotfiles from a user's home directory—but it is coping with
+    # a bunch of applications doing moderately demanding stuff all at once.
+
+    # This is also a pretty good, slightly pessimistic approximation of a busy,
+    # multi-user system like a NAS, which needs to handle multiple 1MB operations
+    # simultaneously for different users. If several people or processes are trying
+    # to read or write big files (photos, movies, whatever) at once, the OS tries
+    # to feed them all data simultaneously. This pretty quickly devolves down to a
+    # pattern of multiple random small block access. So in addition to "busy desktop
+    # with lots of apps," think "busy fileserver with several people actively using it."
+
+    # You will see a lot more variation in speed as you watch this operation play
+    # out on the console. For example, the 4K single process test we tried first
+    # wrote a pretty consistent 11MiB/sec on my MacBook Air's internal drive—but
+    # this 16-process job fluctuated between about 10MiB/sec and 300MiB/sec during
+    # the run, finishing with an average of 126MiB/sec.
+
+    # Most of the variation you're seeing here is due to the operating system and
+    # SSD firmware sometimes being able to aggregate multiple writes. When it
+    # manages to aggregate them helpfully, it can write them in a way that allows
+    # parallel writes to all the individual physical media stripes inside the SSD.
+    # Sometimes, it still ends up having to give up and write to only a single
+    # physical media stripe at a time—or a garbage collection or other maintenance
+    # operation at the SSD firmware level needs to run briefly in the background,
+    # slowing things down.
+    echo ""
+    echo "Parallel 64K Random Writes (size=1G, time=15sec, jobs=16, iodepth=16)"
+    x=`sudo fio \
+        --name=fio-write-random-64k \
+        --directory=$path \
+        --ioengine=posixaio \
+        --rw=randwrite \
+        --bs=64k \
+        --size=64m \
+        --numjobs=16 \
+        --iodepth=16 \
+        --time_based --runtime=15 \
+        --end_fsync=1`
+    echo "  - $x " | /usr/bin/grep -A1 'Run status group' | tail -n1
+
+    # Cleanup my test files
+    rm -rf $path/fio-write-random-64k*
+}
+
+function fio_write_single_sequential_1m {
+    # Single 1M Random Writes
+
+    # This is pretty close to the best-case scenario for a real-world system
+    # doing real-world things. No, it's not quite as fast as a single, truly
+    # contiguous write... but the 1MiB blocksize is large enough that it's quite
+    # close. Besides, if literally any other disk activity is requested simultaneously
+    # with a contiguous write, the "contiguous" write devolves to this level of
+    # performance pretty much instantly, so this is a much more realistic test of
+    # the upper end of storage performance on a typical system.
+
+    # You'll see some kooky fluctuations on SSDs when doing this test. This is largely
+    # due to the SSD's firmware having better luck or worse luck at any given time,
+    # when it's trying to queue operations so that it can write across all physical
+    # media stripes cleanly at once. Rust disks will tend to provide a much more
+    # consistent, though typically lower, throughput across the run.
+
+    # You can also see SSD performance fall off a cliff here if you exhaust an
+    # onboard write cache—TLC and QLC drives tend to have small write cache areas
+    # made of much faster MLC or SLC media. Once those get exhausted, the disk has
+    # to drop to writing directly to the much slower TLC/QLC media where the data
+    # eventually lands. This is the major difference between, for example, Samsung
+    # EVO and Pro SSDs—the EVOs have slow TLC media with a fast MLC cache, where
+    # the Pros use the higher-performance, higher-longevity MLC media throughout
+    # the entire SSD.
+
+    # If you have any doubt at all about a TLC or QLC disk's ability to sustain
+    # heavy writes, you may want to experimentally extend your time duration here.
+    # If you watch the throughput live as the job progresses, you'll see the impact
+    # immediately when you run out of cache—what had been a fairly steady,
+    # several-hundred-MiB/sec throughput will suddenly plummet to half the speed
+    # or less and get considerably less stable as well.
+
+    # However, you might choose to take the opposite position—you might not
+    # expect to do sustained heavy writes very frequently, in which case you
+    # actually are more interested in the on-cache behavior. What's important
+    # here is that you understand both what you want to test, and how to test
+    # it accurately.
+
+    echo ""
+    echo "Single 1M Sequential Writes (size=1G, time=15sec, jobs=1, iodepth=1)"
+    x=`sudo fio \
+        --name=fio-write-random-1m \
+        --directory=$path \
+        --ioengine=posixaio \
+        --rw=write \
+        --bs=1m \
+        --size=1g \
+        --numjobs=1 \
+        --iodepth=1 \
+        --time_based --runtime=15 \
+        --end_fsync=1`
+    echo "  - $x " | /usr/bin/grep -A1 'Run status group' | tail -n1
+
+    # Cleanup my test files
+    rm -rf $path/fio-write-random-1m*
+}
+
+function fio_read_sequential_1m {
+    # Sequential Parallel Reads
+
+    echo ""
+    echo "Sequential 4x 1M Reads"
+    x=`sudo fio \
+        --name=fio-read-sequential-1m \
+        --directory=$path \
+        --ioengine=posixaio \
+        --bs=1M \
+        --numjobs=4 \
+        --size=256M \
+        --time_based --runtime=30s \
+        --ramp_time=2s \
+        --direct=1 \
+        --verify=0 \
+        --iodepth=64 \
+        --rw=read \
+        --group_reporting=1 \
+        --iodepth_batch_submit=64 \
+        --iodepth_batch_complete_max=64`
+    echo "  - $x " | /usr/bin/grep -A1 'Run status group' | tail -n1
+    rm -rf $path/fio-read-sequential-1m*
+}
+
+function fio_read_random_4k {
+    # Random 4k Reads
+
+    echo ""
+    echo "Random 4k Reads"
+    x=`sudo fio \
+        --name=fio-read-random-4k \
+        --directory=$path \
+        --ioengine=posixaio \
+        --rw=randread \
+        --bs=4k \
+        --size=1g \
+        --time_based --runtime=30s \
+        --ramp_time=2s \
+        --direct=1 \
+        --verify=0 \
+        --iodepth=256 \
+        --rw=read \
+        --group_reporting=1 \
+        --iodepth_batch_submit=256 \
+        --iodepth_batch_complete_max=256`
+    echo "  - $x " | /usr/bin/grep -A1 'Run status group' | tail -n1
+    rm -rf $path/fio-read-random-4k*
+}
+
+function fio_speedtest {
+    # Write tests
+    fio_write_single_random_4k
+    fio_write_parallel_random_64k
+    fio_write_single_sequential_1m
+
+    # Read Tests
+    fio_read_sequential_1m
+    fio_read_random_4k
+}
+
+function dd_speedtest {
     # Basic HD speed test using DD
     # mReschke 2017-07-11
     
-file=./bigfile
+    file=$path/bigfile
     size=1024
 
+    echo "Running dd based HD/SSD/NVMe Benchmarks"
+    echo "---------------------------------------"
+
     printf "Cached write speed...\n"
     dd if=/dev/zero of=$file bs=1M count=$size
 
@@ -13,7 +298,7 @@ printf "\nUncached write speed...\n"
     dd if=/dev/zero of=$file bs=1M count=$size conv=fdatasync,notrunc
 
     printf "\nUncached read speed...\n"
-echo 3 > /proc/sys/vm/drop_caches
+    echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null
     dd if=$file of=/dev/null bs=1M count=$size
 
     printf "\nCached read speed...\n"
@@ -21,3 +306,33 @@ dd if=$file of=/dev/null bs=1M count=$size
 
     rm $file
     printf "\nDone\n"
+}
+
+# Show help and usage information
+function usage {
+    echo "Robust Flexible Input/Output HD Speedtest"
+    echo "  If FIO is installed, we use FIO for more detailed performance analysis."
+    echo "  If FIO is not installed, we use basic DD analysis."
+    echo "  You should apt install fio (pacman -S fio) for detailed analysis."
+    echo "mReschke 2024-01-18"
+    echo ""
+    echo "NOTICE, this creates a 1GB file on the desired destination disk."
+    echo "Please ensure you have write access with 1GB free space on destination."
+    echo ""
+    echo "Usage:"
+    echo "  This will use FIO if installed, else DD"
+    echo "    ./speedtest-hd /mnt/somedisk"
+    echo "    ./speedtest-hd ."
+    echo ""
+    echo "  This will force FIO"
+    echo "    ./speedtest-hd /mnt/somedisk --fio"
+    echo "    ./speedtest-hd . --fio"
+    echo ""
+    echo "  This will force DD"
+    echo "    ./speedtest-hd /mnt/somedisk --dd"
+    echo "    ./speedtest-hd . --dd"
+    exit 0
+}
+
+# Go
+main

server/files/profile.d/bash_aliases.sh

@@ -50,8 +50,8 @@ alias pips='pipenv shell && echo "pipenv shell has been deactivated" && echo &&
 alias pops='poetry shell && echo "poetry shell has been deactivated" && echo && pv'
 
 # Docker
-alias dps="docker ps"
+alias dps='docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Networks}}\t{{.Status}}"'
-alias dpsa="docker ps -a"
+alias dpsa='docker ps -a'
-alias dnet="docker network ls"
+alias dnet='docker network ls'
-alias dimg="docker images"
+alias dimg='docker images'
-alias dimga="docker images -a"
+alias dimga='docker images -a'

server/tasks/Agnostic/main.yml

@@ -4,10 +4,10 @@
 # ------------------------------------------------------------------------------
 
 # Copy profiles to /etc/profile.d/
-- include_tasks: ../../../../functions/copy_etc-profile.d.yml
+- include_tasks: ../../../shared/functions/copy_etc-profile.d.yml
 
 # Copy scripts to /usr/local/bin
-- include_tasks: ../../../../functions/copy_usr-local-bin.yml
+- include_tasks: ../../../shared/functions/copy_usr-local-bin.yml
 
 # Rsync /etc/vim
 - name: Synchronizing /etc/vim

server/tasks/Debian/Debian_11.yml

@@ -0,0 +1,71 @@
+---
+
+# Copy sources.list
+- name: Copying {{ ansible_distribution }} {{ ansible_distribution_major_version }} /etc/apt/sources.list
+  copy:
+    src: files/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/sources.list
+    dest: /etc/apt/sources.list
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Ignore apt translations
+- name: Ignoring {{ ansible_distribution }} {{ ansible_distribution_major_version }} APT Translations
+  copy:
+    src: files/{{ ansible_distribution }}/99translations
+    dest: /etc/apt/apt.conf.d/99translations
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Physical Server
+- name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Physical Server
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - ifenslave         # NIC bonding and LACP
+      - bridge-utils      # NIC bridging
+      - vlan              # NIC VLAN tagging
+      - ethtool           # LAN speed and blinking
+      - firmware-linux    # Install all firmware including nonfree and misc
+      - kpartx            # To examine subpartitions of VM images
+      - ntfs-3g           # To examine NTFS VM images
+      - smartmontools     # S.M.A.R.T hard drive tools
+  when: type == 'physical'
+
+# Virtual Machine
+# N/A - name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Virtual Machine
+# N/A   when: type == 'virtual'
+
+# Install common apps (physical or virtual)
+- name: Installing {{ ansible_distribution }} {{ ansible_distribution_major_version }} Applications
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https # For https apt repos
+      - openssh-server      # SSH server
+      - sudo                # Sudo access for users
+      - net-tools           # Deprecated ifconfig
+      - nmap                # Port scanner
+      - ethtool             # LAN information for debugging
+      - iperf               # Network performance for debugging
+      - sysstat             # Performance stats
+      - iotop               # Data I/O top (for hard drive read/write analysis)
+      - iftop               # Network top to watch network usage
+      - vim                 # Vim text editor
+      - nano                # Nano text editor
+      - rsync               # Rsync protocol
+      - curl                # Web downloader
+      - wget                # Web downloader
+      - htop                # Graphical top alternative
+      - mlocate             # Full-text search for all files
+      - zip                 # Zip utility
+      - unzip               # Unzip utility
+      - gzip                # Gzip utility
+      - git                 # Git source control
+      - nfs-common          # Connect to a NFS server
+      - cifs-utils          # Connect to a Samba server
+      - dos2unix            # Convert dos line endings to unix and visa versa
+      - acl                 # I customize directories often with ACL
+      - dnsutils            # Dig command and other dns commands
+      - ntp                 # Date sync

server/tasks/Debian/Debian_12.yml

@@ -0,0 +1,71 @@
+---
+
+# Copy sources.list
+- name: Copying {{ ansible_distribution }} {{ ansible_distribution_major_version }} /etc/apt/sources.list
+  copy:
+    src: files/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/sources.list
+    dest: /etc/apt/sources.list
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Ignore apt translations
+- name: Ignoring {{ ansible_distribution }} {{ ansible_distribution_major_version }} APT Translations
+  copy:
+    src: files/{{ ansible_distribution }}/99translations
+    dest: /etc/apt/apt.conf.d/99translations
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Physical Server
+- name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Physical Server
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - ifenslave         # NIC bonding and LACP
+      - bridge-utils      # NIC bridging
+      - vlan              # NIC VLAN tagging
+      - ethtool           # LAN speed and blinking
+      - firmware-linux    # Install all firmware including nonfree and misc
+      - kpartx            # To examine subpartitions of VM images
+      - ntfs-3g           # To examine NTFS VM images
+      - smartmontools     # S.M.A.R.T hard drive tools
+  when: type == 'physical'
+
+# Virtual Machine
+# N/A - name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Virtual Machine
+# N/A   when: type == 'virtual'
+
+# Install common apps (physical or virtual)
+- name: Installing {{ ansible_distribution }} {{ ansible_distribution_major_version }} Applications
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https # For https apt repos
+      - openssh-server      # SSH server
+      - sudo                # Sudo access for users
+      - net-tools           # Deprecated ifconfig
+      - nmap                # Port scanner
+      - ethtool             # LAN information for debugging
+      - iperf               # Network performance for debugging
+      - sysstat             # Performance stats
+      - iotop               # Data I/O top (for hard drive read/write analysis)
+      - iftop               # Network top to watch network usage
+      - vim                 # Vim text editor
+      - nano                # Nano text editor
+      - rsync               # Rsync protocol
+      - curl                # Web downloader
+      - wget                # Web downloader
+      - htop                # Graphical top alternative
+      - mlocate             # Full-text search for all files
+      - zip                 # Zip utility
+      - unzip               # Unzip utility
+      - gzip                # Gzip utility
+      - git                 # Git source control
+      - nfs-common          # Connect to a NFS server
+      - cifs-utils          # Connect to a Samba server
+      - dos2unix            # Convert dos line endings to unix and visa versa
+      - acl                 # I customize directories often with ACL
+      - dnsutils            # Dig command and other dns commands
+      - ntp                 # Date sync

server/tasks/Debian/Debian_13.yml

@@ -0,0 +1,71 @@
+---
+
+# Copy sources.list
+- name: Copying {{ ansible_distribution }} {{ ansible_distribution_major_version }} /etc/apt/sources.list
+  copy:
+    src: files/{{ ansible_distribution }}/{{ ansible_distribution_major_version }}/sources.list
+    dest: /etc/apt/sources.list
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Ignore apt translations
+- name: Ignoring {{ ansible_distribution }} {{ ansible_distribution_major_version }} APT Translations
+  copy:
+    src: files/{{ ansible_distribution }}/99translations
+    dest: /etc/apt/apt.conf.d/99translations
+    mode: 0644
+  when: modify_apt_sources == True
+
+# Physical Server
+- name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Physical Server
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - ifenslave         # NIC bonding and LACP
+      - bridge-utils      # NIC bridging
+      - vlan              # NIC VLAN tagging
+      - ethtool           # LAN speed and blinking
+      - firmware-linux    # Install all firmware including nonfree and misc
+      - kpartx            # To examine subpartitions of VM images
+      - ntfs-3g           # To examine NTFS VM images
+      - smartmontools     # S.M.A.R.T hard drive tools
+  when: type == 'physical'
+
+# Virtual Machine
+# N/A - name: Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Virtual Machine
+# N/A   when: type == 'virtual'
+
+# Install common apps (physical or virtual)
+- name: Installing {{ ansible_distribution }} {{ ansible_distribution_major_version }} Applications
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https # For https apt repos
+      - openssh-server      # SSH server
+      - sudo                # Sudo access for users
+      - net-tools           # Deprecated ifconfig
+      - nmap                # Port scanner
+      - ethtool             # LAN information for debugging
+      - iperf               # Network performance for debugging
+      - sysstat             # Performance stats
+      - iotop               # Data I/O top (for hard drive read/write analysis)
+      - iftop               # Network top to watch network usage
+      - vim                 # Vim text editor
+      - nano                # Nano text editor
+      - rsync               # Rsync protocol
+      - curl                # Web downloader
+      - wget                # Web downloader
+      - htop                # Graphical top alternative
+      - plocate             # Full-text search for all files (a faster mlocate)
+      - zip                 # Zip utility
+      - unzip               # Unzip utility
+      - gzip                # Gzip utility
+      - git                 # Git source control
+      - nfs-common          # Connect to a NFS server
+      - cifs-utils          # Connect to a Samba server
+      - dos2unix            # Convert dos line endings to unix and visa versa
+      - acl                 # I customize directories often with ACL
+      - dnsutils            # Dig command and other dns commands
+      # NOTE, ntp is no longer used for Debian 13 as it uses systemd-timesyncd

server/tasks/Debian/all-OBSOLETE.yml

@@ -6,28 +6,38 @@
 # Copy Debian 9 sources.list
 - name: Copying Debian 9 apt/sources.list
   copy: src=debian/9/sources.list dest=/etc/apt/sources.list mode=0644
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "9"
+  when: modify_apt_sources == True and ansible_os_family == "Debian" and ansible_distribution_major_version == "9"
 
 # Copy Debian 10 sources.list
 - name: Copying Debian 10 apt/sources.list
   copy: src=debian/10/sources.list dest=/etc/apt/sources.list mode=0644
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
+  when: modify_apt_sources == True and ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
 
 # Copy Debian 11 sources.list
 - name: Copying Debian 11 apt/sources.list
   copy: src=debian/11/sources.list dest=/etc/apt/sources.list mode=0644
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "11"
+  when: modify_apt_sources == True and ansible_os_family == "Debian" and ansible_distribution_major_version == "11"
+
+# Copy Debian 12 sources.list
+- name: Copying Debian 12 apt/sources.list
+  copy: src=debian/12/sources.list dest=/etc/apt/sources.list mode=0644
+  when: modify_apt_sources == True and ansible_os_family == "Debian" and ansible_distribution_major_version == "12"
+
+# Copy Debian 13 sources.list
+- name: Copying Debian 13 apt/sources.list
+  copy: src=debian/13/sources.list dest=/etc/apt/sources.list mode=0644
+  when: modify_apt_sources == True and ansible_os_family == "Debian" and ansible_distribution_major_version == "13"
 
   # Copy Ubuntu 16.04 sources.list
 - name: Copying Ubuntu 16.04 apt/sources.list
   copy: src=ubuntu/16.04/sources.list dest=/etc/apt/sources.list mode=0644
-  when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "16.04"
+  when: modify_apt_sources == True and ansible_distribution == "Ubuntu" and ansible_distribution_version == "16.04"
 
 # Ignore apt translations
 - name: Ignoring apt tranlations
   copy: src=debian/99translations dest=/etc/apt/apt.conf.d/99translations mode=0644
 
-# Install common apps for all debian machines
+# Install common apps for all Debian and Ubuntu Systems
 - name: Installing common Debian/Ubuntu applications
   apt:
     update_cache: yes
@@ -50,7 +60,6 @@
       - wget                # Web downloader
       - htop                # Graphical top alternative
       - mlocate             # Full-text search for all files
-      - ntp                 # Date sync
       - zip                 # Zip utility
       - unzip               # Unzip utility
       - gzip                # Gzip utility
@@ -60,3 +69,15 @@
       - dos2unix            # Convert dos line endings to unix and visa versa
       - acl                 # I customize directories often with ACL
       - dnsutils            # Dig command and other dns commands
+
+
+# Install common apps for all Debian <= 11 and Ubunt <= 22.04
+- name: Installing common Debian <=11 and Ubuntu <= 22.04 applications
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - ntp                 # Date sync
+  when: (ansible_distribution == "Ubuntu" and ansible_distribution_version <= "22.04") or
+        (ansible_distribution == "Debian" and ansible_distribution_major_version <= "11")
+

server/tasks/Debian/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Dynamically include proper file based on OS and Version
+- name: "Configuring {{ ansible_distribution }} {{ ansible_distribution_major_version }} Server (physical or virtual)"
+  include_tasks: "{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml"

server/tasks/Ubuntu/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Dynamically include proper file based on OS and Version
+- name: Configuring Debian Server (physical or virtual)
+  include_tasks: "{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml"

server/tasks/debian/main.yml

@@ -1,19 +0,0 @@
----
-# Debian Server
-# ------------------------------------------------------------------------------
-- name: Configuring Debian/Ubuntu physical OR virtual server
-  include_tasks: all.yml
-
-# Physical Debian Server
-# ------------------------------------------------------------------------------
-- name: Configuring Debian/Ubuntu physical server
-  include_tasks: physical.yml
-  when: type == 'physical'
-
-
-# Virtual Debian Server
-# ------------------------------------------------------------------------------
-- name: Configuring Debian/Ubuntu virtual server
-  include_tasks: virtual.yml
-  when: type == 'virtual'
-

server/tasks/debian/physical.yml

@@ -1,19 +0,0 @@
----
-# ------------------------------------------------------------------------------
-# These tasks run for physical Debian/Ubuntu servers
-# ------------------------------------------------------------------------------
-
-# Physical Debian/Ubuntu Server
-- name: Installing physical Debian/Ubuntu server applications
-  apt:
-    update_cache: yes
-    state: present
-    name:
-      - ifenslave         # NIC bonding and LACP
-      - bridge-utils      # NIC bridging
-      - vlan              # NIC VLAN tagging
-      - ethtool           # LAN speed and blinking
-      - firmware-linux    # Install all firmware including nonfree and misc
-      - kpartx            # To examine subpartitions of VM images
-      - ntfs-3g           # To examine NTFS VM images
-      - smartmontools     # S.M.A.R.T hard drive tools

server/tasks/debian/virtual.yml

@@ -1,5 +0,0 @@
----
-# ------------------------------------------------------------------------------
-# These tasks run for virtual Debian/Ubuntu servers
-# ------------------------------------------------------------------------------
-

server/tasks/main.yml

@@ -1,19 +1,9 @@
 ---
-# Debian server
-# Run first to install base software required for agnostic/main.yml
-# ------------------------------------------------------------------------------
-- name: Configure debian/ubuntu server
-  include_tasks: debian/main.yml
-  when: ansible_os_family == "Debian"
 
-# Manjaro server
+# OS Specific main.yml
-# Run first to install base software required for agnostic/main.yml
+- name: "Configuring {{ ansible_distribution }} System"
-# ------------------------------------------------------------------------------
+  include_tasks: "{{ ansible_distribution }}/main.yml"
-- name: Configure Manjaro server
-  include_tasks: manjaro/main.yml
-  when: ansible_os_family == "Archlinux" and ansible_lsb.id == "ManjaroLinux"
 
 # Any server (OS agnostic)
-# ------------------------------------------------------------------------------
+- name: Configuring OS Agnostic System
-- name: Configure any server (OS agnostic)
+  include_tasks: Agnostic/main.yml
-  include_tasks: agnostic/main.yml

user/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+user_groups: ""
+add_groups: ""
+ssh_into: ""
+home: ""
+add_sudo: false
+create: true
+ssh_keys: false
+ssh_authorize: true
+shell: /bin/bash
+create_home: true

user/tasks/authorize.yml

@@ -3,5 +3,5 @@
 - name: Adding {{ user }} SSH key to {{ ssh_into }} users ~/.ssh/authorized_keys
   authorized_key:
     user: '{{ item }}'
-    key: "{{ lookup('file', user_path + '/keys/' + user + '.key.pub') }}"
+    key: "{{ lookup('file', '../roles/user/keys/' + user + '.key.pub') }}"
   with_items: '{{ ssh_into }}'

user/tasks/create.yml

@@ -17,7 +17,7 @@
 # Add OS specific sudo group
 - set_fact:
     user_groups: "{{ user_groups + [sudogroup] }}"
-  when: add_sudo|default(false)|bool
+  when: add_sudo == true
 
 # Set root groups
 - set_fact:
@@ -25,9 +25,14 @@
   when: user == 'root'
 
 # Home directory, use /home/username if undefined
-# - set_fact:
+- set_fact:
-#     home: "/home/{{ user }}"
+    home: "/home/{{ user }}"
-#   when: (home is undefined or home == "")
+  when: (home is undefined or home == "")
+
+# Override home variable if user is root, only allow /root
+- set_fact:
+    home: "/root"
+  when: (user == 'root')
 
 # ------------------------------------------------------------------------------
 
@@ -43,10 +48,10 @@
 - set_fact:
     user_groups: ""
     add_groups: ""
-    ssh_info: ""
+    ssh_into: ""
     home: ""
-    add_sudo: no
+    add_sudo: false
-    create: yes
+    create: true
-    ssh_keys: no
+    ssh_keys: false
-    ssh_authorize: yes
+    ssh_authorize: true
 

user/tasks/main.yml

@@ -28,11 +28,11 @@
 ################################################################################
 # Create user and groups
 - include_tasks: create.yml
-  when: create|default(true)|bool
+  when: create == true
 
 # Authorize user via SSH
 - include_tasks: authorize.yml
-  when: ssh_into is defined and ssh_info != ""
+  when: ssh_into is defined and ssh_into != ""
 
 # Debian modifications per user
 - include_tasks: debian.yml

user/tasks/manjaro.yml

@@ -7,7 +7,7 @@
     create: yes
   when: user != 'root'
 
-- # Manjaro symlink ~/.vim
+# Manjaro symlink ~/.vim
 - name: Symlinking ~/.vim to /etc/vim
   file:
     src: /etc/vim

user/tasks/ssh.yml

@@ -15,19 +15,18 @@
     user: '{{ user }}'
     key: '{{ item }}'
   with_file:
-    - '{{ user_path }}/keys/{{ user }}.key.pub'
+    - '../roles/user/keys/{{ user }}.key.pub'
-  when: ssh_authorize|default(true)|bool
+  when: ssh_authorize == true
 
 # Create users public key
 - name: Copying {{ user }} SSH public key
   copy:
-    src: '{{ user_path }}/keys/{{ user }}.key.pub'
+    src: '../roles/user/keys/{{ user }}.key.pub'
-    #dest: '{{ user_home }}/.ssh/id_rsa.pub'
     dest: '{{ "~" + user | expanduser }}/.ssh/id_rsa.pub'
     owner: '{{ user }}'
     group: '{{ user }}'
     mode: '0644'
-  when: ssh_keys|default(false)|bool
+  when: ssh_keys == true
 
 # Create users private key
 - name: Copying {{ user }} SSH private key
@@ -37,4 +36,4 @@
     owner: '{{ user }}'
     group: '{{ user }}'
     mode: '0600'
-  when: ssh_keys|default(false)|bool
+  when: ssh_keys == true

user/tasks/user.yml

@@ -16,5 +16,5 @@
     password: "{{ users[user]['password'] }}"
     update_password: always
     home: "{{ home }}"
-    create_home: "{{ create_home | default('yes') }}"
+    create_home: "{{ create_home }}"
-    shell: "{{ shell | default('/bin/bash') }}"
+    shell: "{{ shell }}"

virt/arcane/files/compose.yml

@@ -0,0 +1,20 @@
+services:
+  arcane:
+    image: ghcr.io/getarcaneapp/arcane:latest
+    container_name: arcane
+    ports:
+      - '3552:3552'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - arcane-data:/app/data
+      - /opt/projects:/app/data/projects
+    environment:
+      - APP_URL=http://arcane1.mreschke.net:3552
+      - PUID=1000
+      - PGID=1000
+      - ENCRYPTION_KEY=FbAK6DaqI1EEA5bEWPyBLetsuY748DbqWzw4a3/q3G0=
+      - JWT_SECRET=flLR6aTtJSQ+PgxlHW5pUrE6LlBwXxbnvwy5g4dFUS0=
+    restart: unless-stopped
+
+volumes:
+  arcane-data:

virt/arcane/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+
+# Create /opt/arcane
+- name: Creating /opt/arcane
+  file:
+    path: /opt/arcane
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+# Create /opt/projects
+- name: Creating /opt/projects
+  file:
+    path: /opt/projects
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+# Copy Arcane docker compose file
+- name: Copying Arcane compose.yml
+  copy:
+    src: compose.yml
+    dest: /opt/arcane/compose.yml
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    mode: '0644'
+
+# Docker compose up
+- name: Starting Arcane if not already running
+  shell: "docker compose up -d"
+  args:
+    chdir: /opt/arcane

virt/docker-OLD/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+- name: Installing Docker apt repository prerequisites
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      - software-properties-common
+
+- name: Addding Debian Docker apt repository key
+  apt_key: url='https://download.docker.com/linux/debian/gpg' state=present
+
+- name: Adding Debian 9 Docker apt repository sources
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian stretch stable' state=present
+  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "9"
+
+- name: Adding Debian 10 Docker apt repository sources
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian buster stable' state=present
+  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
+
+- name: Adding Debian 11 Docker apt repository sources
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bullseye stable' state=present
+  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "11"
+
+- name: Adding Debian 12 Docker apt repository sources
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bookworm stable' state=present
+  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "12"
+
+- name: Installing Docker
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-compose

virt/docker/defaults/main.yml

@@ -0,0 +1,14 @@
+docker_data_root: "/var/lib/docker"
+
+# A 172.16.0.0/16 gives max of 172.16.255.254
+# And a /25 pool size means 126 IPs in each docker network
+docker_default_address_pool_base: "172.16.0.0/16"
+docker_default_address_pool_size: "25"
+
+# Log Opts Defaults https://docs.docker.com/config/containers/logging/json-file/
+# NOTE: integers must be quoted as strings or you get this error "cannot unmarshal number into Go struct field Config.log-opts of type string"
+docker_log_driver: "json-file"
+docker_log_opts_max_size: "100m"
+docker_log_opts_max_file: "3"
+docker_version: 28.4.0
+

virt/docker/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart docker
+  service:
+    name: docker
+    state: restarted

virt/docker/tasks/Debian/Debian_11.yml

@@ -0,0 +1,81 @@
+---
+
+# Add some prerequisites
+- name: Installing Docker Prerequisites on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      - software-properties-common
+
+# Add APT GPG key
+- name: Adding Docker APT repository key for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt_key: url='https://download.docker.com/linux/debian/gpg' state=present
+
+# Add apt repository
+- name: Adding Docker APT repository for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bullseye stable' state=present
+
+# Apt update
+- name: Updating apt cache
+  apt:
+    update_cache: yes
+
+# Get exact apt package docker version based on var docker_version
+- name: Finding APT Docker Version String for {{ docker_version }}
+  shell: "apt-cache madison docker-ce | awk '{ print $3 }' | grep '{{ docker_version }}' | sort | tail -n1"
+  register: docker_version_output
+
+# Setting docker version string
+- name: Set docker_version_string variable to {{ docker_version_output.stdout }}
+  set_fact:
+    docker_version_string: "{{ docker_version_output.stdout }}"
+
+# Install docker and docker-compose-plugin
+# note that with the version in here it will upgrade docker to this version, it may fail if a newer version is installed
+- name: Installing Docker
+  apt:
+    update_cache: yes
+    state: present
+    # Tell apt to NOT start docker after install
+    policy_rc_d: 101
+    name:
+      - docker-ce={{ docker_version_string }}
+      - docker-ce-cli={{ docker_version_string }}
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+
+# Configure docker /etc/docker/daemon.js (notice docker has NOT started yet because of policy_rc_d)
+- name: Creating /etc/docker
+  file:
+    path: /etc/docker
+    owner: root
+    group: root
+    state: directory
+- name: Templating /etc/daemon.json
+  template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    mode: 0644
+    backup: yes
+  notify: restart docker
+
+# Start docker now that we have copied our custom /etc/docker/daemon.json
+- name: Starting docker daemon
+  service:
+    name: docker
+    enabled: yes
+    state: started
+
+# Install python-docker (required for ansible docker modules)
+- name: Installing python3-docker for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name: python3-docker

virt/docker/tasks/Debian/Debian_12.yml

@@ -0,0 +1,81 @@
+---
+
+# Add some prerequisites
+- name: Installing Docker Prerequisites on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      - software-properties-common
+
+# Add APT GPG key
+- name: Adding Docker APT repository key for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt_key: url='https://download.docker.com/linux/debian/gpg' state=present
+
+# Add apt repository
+- name: Adding Docker APT repository for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bookworm stable' state=present
+
+# Apt update
+- name: Updating apt cache
+  apt:
+    update_cache: yes
+
+# Get exact apt package docker version based on var docker_version
+- name: Finding APT Docker Version String for {{ docker_version }}
+  shell: "apt-cache madison docker-ce | awk '{ print $3 }' | grep '{{ docker_version }}' | sort | tail -n1"
+  register: docker_version_output
+
+# Setting docker version string
+- name: Set docker_version_string variable to {{ docker_version_output.stdout }}
+  set_fact:
+    docker_version_string: "{{ docker_version_output.stdout }}"
+
+# Install docker and docker-compose-plugin
+# note that with the version in here it will upgrade docker to this version, it may fail if a newer version is installed
+- name: Installing Docker
+  apt:
+    update_cache: yes
+    state: present
+    # Tell apt to NOT start docker after install
+    policy_rc_d: 101
+    name:
+      - docker-ce={{ docker_version_string }}
+      - docker-ce-cli={{ docker_version_string }}
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+
+# Configure docker /etc/docker/daemon.js (notice docker has NOT started yet because of policy_rc_d)
+- name: Creating /etc/docker
+  file:
+    path: /etc/docker
+    owner: root
+    group: root
+    state: directory
+- name: Templating /etc/daemon.json
+  template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    mode: 0644
+    backup: yes
+  notify: restart docker
+
+# Start docker now that we have copied our custom /etc/docker/daemon.json
+- name: Starting docker daemon
+  service:
+    name: docker
+    enabled: yes
+    state: started
+
+# Install python-docker (required for ansible docker modules)
+- name: Installing python3-docker for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name: python3-docker

virt/docker/tasks/Debian/Debian_13.yml

@@ -0,0 +1,204 @@
+---
+
+# Add some prerequisites
+- name: Installing Docker Prerequisites on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      #- software-properties-common # not on debian 13
+
+# Add APT GPG key
+# NOTE apt_key is deprecated in Debian 13, use get_url and store the key in /etc/apt/trusted.gpg.d/*
+- name: Adding Docker APT repository key for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  ansible.builtin.get_url:
+    url: "https://download.docker.com/linux/debian/gpg"
+    dest: /etc/apt/trusted.gpg.d/docker.asc
+    mode: '0644'
+    force: true
+
+# Add apt repository
+- name: Adding Docker APT repository for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt_repository:
+    filename: docker
+    state: present
+    repo: 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/debian trixie stable'
+
+# Apt update
+- name: Updating apt cache
+  apt:
+    update_cache: yes
+
+# Get exact apt package docker version based on var docker_version
+- name: Finding APT Docker Version String for {{ docker_version }}
+  shell: "apt-cache madison docker-ce | awk '{ print $3 }' | grep '{{ docker_version }}' | sort | tail -n1"
+  register: docker_version_output
+
+# Setting docker version string
+- name: Set docker_version_string variable to {{ docker_version_output.stdout }}
+  set_fact:
+    docker_version_string: "{{ docker_version_output.stdout }}"
+
+# Install docker and docker-compose-plugin
+# note that with the version in here it will upgrade docker to this version, it may fail if a newer version is installed
+- name: Installing Docker
+  apt:
+    update_cache: yes
+    state: present
+    # Tell apt to NOT start docker after install
+    policy_rc_d: 101
+    name:
+      - docker-ce={{ docker_version_string }}
+      - docker-ce-cli={{ docker_version_string }}
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+
+# Configure docker /etc/docker/daemon.js (notice docker has NOT started yet because of policy_rc_d)
+- name: Creating /etc/docker
+  file:
+    path: /etc/docker
+    owner: root
+    group: root
+    state: directory
+- name: Templating /etc/daemon.json
+  template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    mode: 0644
+    backup: yes
+  notify: restart docker
+
+# Start docker now that we have copied our custom /etc/docker/daemon.json
+- name: Starting docker daemon
+  service:
+    name: docker
+    enabled: yes
+    state: started
+
+# Install python-docker (required for ansible docker modules)
+- name: Installing python3-docker for {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  apt:
+    update_cache: yes
+    state: present
+    name: python3-docker
+
+
+
+
+# # Add docker apt GPG key and repo (creates to /etc/apt/sources.list.d/download_docker_com_linux_debian.list )
+# - name: Adding Docker apt repository key
+#   apt_key: url='https://download.docker.com/linux/debian/gpg' state=present
+#   when: ansible_distribution == 'Debian'
+
+# - name: Adding Docker apt repository key
+#   apt_key: url='https://download.docker.com/linux/ubuntu/gpg' state=present
+#   when: ansible_distribution == 'Ubuntu'
+
+# - name: Adding Debian 9 Docker apt repository sources
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian stretch stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "9"
+
+# - name: Adding Debian 10 Docker apt repository sources
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian buster stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
+
+# - name: Adding Debian 11 Docker apt repository sources
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bullseye stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "11"
+
+# - name: Adding Debian 12 Docker apt repository sources
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bookworm stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "12"
+
+# - name: Adding Docker apt repository sources for Ubuntu 18
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "18"
+
+# - name: Adding Docker apt repository sources for Ubuntu 20
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "20"
+
+# - name: Adding Docker apt repository sources for Ubuntu 22
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "22"
+
+# - name: Adding Docker apt repository sources for Ubuntu 24
+#   apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/ubuntu noble stable' state=present
+#   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "24"
+
+
+
+# - name: Update apt cache
+#   apt:
+#     update_cache: yes
+
+# - name: find docker version string
+#   shell: "apt-cache madison docker-ce | awk '{ print $3 }' | grep '{{ docker_version }}' | sort | tail -n1"
+#   register: docker_version_output
+
+# - name: set docker_version_string variable
+#   set_fact:
+#     docker_version_string: "{{ docker_version_output.stdout }}"
+
+# # Install docker and docker-compose-plugin
+# # note that with the version in here it will upgrade docker to this version, it may fail if a newer version is installed
+# - name: Installing Docker
+#   apt:
+#     update_cache: yes
+#     state: present
+#     # Tell apt to NOT start docker after install
+#     policy_rc_d: 101
+#     name:
+#       - docker-ce={{ docker_version_string }}
+#       - docker-ce-cli={{ docker_version_string }}
+#       - containerd.io
+#       - docker-buildx-plugin
+#       - docker-compose-plugin
+
+# # Configure docker /etc/docker/daemon.js (notice docker has NOT started yet because of policy_rc_d)
+# # - name: Create /etc/docker
+# #   file:
+# #     path: /etc/docker
+# #     owner: root
+# #     group: root
+# #     state: directory
+
+# # - name: Copying daemon.json
+# #   template:
+# #     src: daemon.json.j2
+# #     dest: /etc/docker/daemon.json
+# #     owner: root
+# #     group: root
+# #     mode: 0644
+# #     backup: yes
+# #   notify: restart docker
+
+
+# # Start docker now that we have copied our custom /etc/docker/daemon.json
+# - name: Starting docker daemon
+#   service:
+#     name: docker
+#     enabled: yes
+#     state: started
+
+
+# # # Install python-docker (required for ansible docker modules)
+# # - name: Installing python-docker for Ubuntu 18
+# #   apt:
+# #     update_cache: yes
+# #     state: present
+# #     name: python-docker
+# #   when: ansible_os_family == "Debian" and ansible_distribution_major_version == "18"
+
+# # - name: Installing python3-docker for Ubuntu 20+
+# #   apt:
+# #     update_cache: yes
+# #     state: present
+# #     name: python3-docker
+# #   when: ansible_os_family == "Debian" and ansible_distribution_major_version >= "20"

virt/docker/tasks/Debian/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Dynamically include proper file based on OS and Version
+- name: Configuring Docker on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  include_tasks: "{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml"

virt/docker/tasks/Ubuntu/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Dynamically include proper file based on OS and Version
+- name: Configuring Docker on {{ ansible_distribution }} {{ ansible_distribution_major_version }}
+  include_tasks: "{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml"

virt/docker/tasks/main.yml

@@ -1,35 +1,5 @@
 ---
-- name: Installing Docker apt repository prerequisites
-  apt:
-    update_cache: yes
-    state: present
-    name:
-      - apt-transport-https
-      - ca-certificates
-      - gnupg2
-      - software-properties-common
 
-- name: Addding Debian Docker apt repository key
+# OS Specific main.yml
-  apt_key: url='https://download.docker.com/linux/debian/gpg' state=present
+- name: "Configure Docker on {{ ansible_distribution }}"
-
+  include_tasks: "{{ ansible_distribution }}/main.yml"
-- name: Adding Debian 9 Docker apt repository sources
-  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian stretch stable' state=present
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "9"
-
-- name: Adding Debian 10 Docker apt repository sources
-  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian buster stable' state=present
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "10"
-
-- name: Adding Debian 11 Docker apt repository sources
-  apt_repository: repo='deb [arch=amd64] https://download.docker.com/linux/debian bullseye stable' state=present
-  when: ansible_os_family == "Debian" and ansible_distribution_major_version == "11"
-
-- name: Installing Docker
-  apt:
-    update_cache: yes
-    state: present
-    name:
-      - docker-ce
-      - docker-ce-cli
-      - containerd.io
-      - docker-compose

virt/docker/templates/daemon.json.j2

@@ -0,0 +1,17 @@
+{
+  "cgroup-parent": "system.slice",
+  "containerd-namespace": "docker",
+  "default-address-pools": [
+    {
+      "base": "{{ docker_default_address_pool_base }}",
+      "size": {{ docker_default_address_pool_size }}
+    }
+  ],
+  "storage-driver": "overlay2",
+  "data-root": "{{ docker_data_root }}",
+  "log-driver": "{{ docker_log_driver }}",
+  "log-opts": {
+    "max-size": "{{ docker_log_opts_max_size }}",
+    "max-file": "{{ docker_log_opts_max_file }}"
+  }
+}

virt/dockge/files/compose.yml

@@ -0,0 +1,13 @@
+services:
+  dockge:
+    image: louislam/dockge:latest
+    restart: unless-stopped
+    ports:
+      - 5001:5001
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /opt/dockge/data:/app/data
+      - /opt/stacks:/opt/stacks
+    environment:
+      # Tell Dockge where to find the stacks
+      - DOCKGE_STACKS_DIR=/opt/stacks

virt/dockge/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+
+# Create /opt/dockge
+- name: Creating /opt/dockge
+  file:
+    path: /opt/dockge
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+# Create /opt/stacks
+- name: Creating /opt/stacks
+  file:
+    path: /opt/stacks
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+
+# Create /opt/dockge
+- name: Creating /opt/dockge
+  file:
+    path: /opt/dockge
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+# Create /opt/dockge/data
+- name: Creating /opt/dockge/data
+  file:
+    path: /opt/dockge/data
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    state: directory
+
+# Copy Dockge docker compose file
+- name: Copying Dockge compose.yml
+  copy:
+    src: compose.yml
+    dest: /opt/dockge/compose.yml
+    owner: "{{ superuser }}"
+    group: "{{ supergroup }}"
+    mode: '0644'
+
+# Docker compose up
+- name: Starting Dockge if not already running
+  shell: "docker compose up -d"
+  args:
+    chdir: /opt/dockge

virt/opennebula-5.4-kvm-node/tasks/configure.yml

@@ -77,7 +77,7 @@
   notify: restart libvirtd # Only runs if file changed!
 
 # Copy scripts to /usr/local/bin
-- include_tasks: ../../../functions/copy_usr-local-bin.yml
+- include_tasks: ../../shared/functions/copy_usr-local-bin.yml
 
 # Schedule cron
 - name: Scheduling cron cron-root-daily-12am

vpn/openvpn/tasks/debian/main.yml

@@ -0,0 +1,8 @@
+---
+# Install OpenVPN on Debian/Ubuntu
+- name: Installing OpenVPN on Debian/Ubuntu
+  apt:
+    update_cache: yes
+    state: present
+    name:
+      - openvpn

vpn/openvpn/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# Debian server
+# Run first to install base software required for agnostic/main.yml
+# ------------------------------------------------------------------------------
+- name: Configure Debian/Ubuntu server
+  include_tasks: debian/main.yml
+  when: ansible_os_family == "Debian"

web/certbot/tasks/main.yml

@@ -6,4 +6,4 @@
     state: present
     name:
       - certbot
-      - python-certbot-nginx
+      - python3-certbot-nginx

web/haproxy/tasks/main.yml

@@ -6,6 +6,14 @@
     state: present
     name: haproxy
 
+# Ensure /etc/haproxy/ssl/certs exists
+- name: Creating /etc/haproxy/ssl/certs
+  file:
+    path: /etc/haproxy/ssl/certs
+    state: directory
+    owner: root
+    group: root
+
 # Copy /etc/rsyslog.conf
 - name: Copying /etc/rsyslog.conf
   copy:
@@ -28,4 +36,4 @@
   notify: restart rsyslog
 
 # Copy scripts to /usr/local/bin
-- include_tasks: ../../../functions/copy_usr-local-bin.yml
+- include_tasks: ../../shared/functions/copy_usr-local-bin.yml

web/nginx/tasks/main.yml

@@ -4,4 +4,4 @@
   apt:
     update_cache: yes
     state: present
-    name: nginx-light
+    name: nginx