Move all shared into this new repo

web/haproxy/files/49-haproxy.conf

@@ -0,0 +1,12 @@
+$ModLoad imudp
+$UDPServerRun 514
+$UDPServerAddress 127.0.0.1
+
+local0.* -/var/log/haproxy_info.log
+local1.* -/var/log/haproxy_status.log
+& ~
+# & ~ means not to put what matched in the above line anywhere else for the rest of the rules
+# http://serverfault.com/questions/214312/how-to-keep-haproxy-log-messages-out-of-var-log-syslog
+
+# Actually found haproxy was being added to /var/log/syslog too, I modified the /etc/rsyslog.conf and set the line *.*;
+# also the line about /var/log/message, append the local0.none and local1.none

web/haproxy/files/bin/haproxy_create_error_log.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Parse haproxy log for yesterday
+# Find all errors codes >= 400
+# Convert output to CSV format and save on xenstore :)
+# mReschke 2012-05-11
+
+yesterday=$(date --date "$dte -1 days" '+%Y-%m-%d')
+log_src=/store/data/Production/log/Xenlb/haproxy/${yesterday}_Log.csv
+log_dest=/store/data/Production/log/Xenlb/haproxy/${yesterday}_ErrorLog.csv
+
+touch /tmp/haproxy_create_error_log.alive
+
+#this was for the old csv log
+#cat $log_src \
+#	| awk -F\",\" '{ if (substr($3,1,1) > 3) { print $0 }}' \
+#	> $log_dest
+
+
+cat $log_src \
+	| awk -F\  '{ if (substr($11,1,1) > 3) { print $0 }}' \
+	> $log_dest
+
+
+chown toor:toor $log_dest

web/haproxy/files/bin/haproxy_filter.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Filter (cut and color) the /var/log/haproxy_info.log
+# Usage: cat /var/log/haproxy_info.log | grep whatever | haproxy_filter
+# mReschke 2012-05-03
+
+# Color Reference, would use vars but can't in awk
+DEFAULT="\033[0;0m"
+BLUE="\033[0;34m"
+GREEN="\033[0;32m"
+CYAN="\033[0;36m"
+RED="\033[0;31m"
+PURPLE="\033[0;35m"
+BROWN="\033[0;33m"
+LIGHTGRAY="\033[0;37m"
+
+DARKGRAY="\033[1;30m"
+LIGHTBLUE="\033[1;34m"
+LIGHTGREEN="\033[1;32m"
+LIGHTCYAN="\033[1;36m"
+LIGHTRED="\033[1;31m"
+LIGHTPURPLE="\033[1;35m"
+YELLOW="\033[1;33m"
+WHITE="\033[1;37m"
+
+# $10 are the timers, the 5th element is the speeda
+# See http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#8.4
+# timers[5] = Tt (total time, this is generally what I use)
+# timers[5] = Tr
+
+# Filter STDIN
+while read data; do
+echo "$data" \
+	| sed 's/  / /g' \
+	| awk '{ printf("\
+\033[0;35m%3s %2s %8s \033[0;33m| ", $1,$2,$3); \
+if (substr($6,1,index($6,":")-1) == "66.196.205.188") {
+	printf("\033[1;37m%15s \033[0;33m| ", substr($6,1,index($6,":")-1));
+} else {
+	printf("\033[0;32m%15s \033[0;33m| ", substr($6,1,index($6,":")-1));
+}
+split($10, timers, "/");
+printf("\033[0;37m%15s \033[0;33m| ", $9); \
+if (substr($11,1,1) == "2") {
+	printf("\033[1;32m%5s ", $11);
+} else if (substr($11,1,1) == "3") {
+	printf("\033[0;34m%5s ", $11);
+} else if (substr($11,1,1) == "4") {
+	printf("\033[1;31m%5s ", $11);
+} else if (substr($11,1,1) == "5") {
+	printf("\033[0;31m%5s ", $11);
+} else {
+	printf("\033[1;37m%5s ", $11);
+}
+printf("\033[0;33m| \
+\033[0;37m%-30s \033[0;33m| \
+\033[0;37m%4s \033[0;33m| \
+\033[0;37m%20s \033[0;33m| \
+\033[0;37m%-4.0f \033[0;33m| \
+\033[0;37m%s\n", \
+substr(substr($18, 2, length($18)-2),1,30), substr($19,2,5),$10,($12/1024),$20)}'
+
+done

web/haproxy/files/bin/haproxy_filter_csv.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Filter /var/log/haproxy_info.log into CSV format
+# mReschke 2012-05-11
+
+#$1,2,3 datetime
+#$6 IP
+#$9 server (backend)
+#$10 is the time /x/x/x/x/x section
+#$11 http response code
+#$12 size in bytes
+#$18 domain
+#$19 method (get, post)
+#$20 URL
+
+
+# Update on 2014-08-11, found this was using the wrong time, it was using a[1] which is Tq but should have been a[4] which is Tt
+# so all csvs time columns is wrong before this date
+
+# Read from stdin
+while read data; do
+	echo "$data" \
+	sed 's/  / /g' \
+	| awk '{ printf("\"%s %s %s\",\"%s\",\"%s\",\"%s\",\"%s\",", $1,$2,$3,substr($6,1,index($6,":")-1),$11,$9,substr($18,2,length($18)-2)); \
+		split($10,a,"/");
+		printf("\"%s\",\"%s\",\"%s\",\"%s\"\n",a[4],$12,substr($19,2,5),$20); }'
+done

web/haproxy/files/bin/haproxy_filter_speed.awk

@@ -0,0 +1,60 @@
+#!/usr/bin/awk -f
+
+BEGIN {
+	FS = " "
+
+	# Output as CSV
+	csv=0
+}
+
+{
+	client=$6
+	date=$7
+	time=substr(date, 14, 12)
+	backend=$9
+	split($10, timers, "/")
+	tt=timers[4]
+	status=$11
+	size=$12 / 1024 #in kb
+	termination=$15
+	split($16, conns, "/")
+	host=substr($18, 2, length($18)-2)
+	subdomain=substr(host, 1, index(host, ".")-1)
+	request=$20
+	page=getPage(request)
+
+
+	#out(date, 26)
+	out(time, 12)
+	#out(substr(host, 1, 30), 30)
+	if (csv == 1)
+		out(subdomain, 20)
+	else
+		out(substr(subdomain, 1, 20), 20)
+	out(status, 3)
+	#out(client, 21)
+	out("["termination"]", 4)
+	out(size, 6.1, "f")
+	out(tt, 5, "d")
+	out(page)
+	
+	printf("\n")
+
+}
+
+function out(data, pad, type) {
+	if (type == "") type = "s"
+	if (csv == 1)
+		printf("%s", "\""data"\",")
+	else
+		printf("%-"pad""type"  ", data)
+}
+
+function getPage(request) {
+	if (index(request, "?") > 0) 
+		return substr(request, 1, index(request, "?")-1)
+	else
+		return request
+
+}
+

web/haproxy/files/bin/watch_haproxy.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Tail haproxy log and filter through my nice cut & color filter
+# Must use stdbuf -oL because output is buffered, so stdbuf -oL allows for a realtime strea of tail
+# mReschke 2012-05-03
+#if [ "$1" == "" ]; then
+	#tail -f /var/log/haproxy_info.log | stdbuf -oL haproxy_filter
+
+	# This filters out web5-https and zabbix hits
+#	tail -f /var/log/haproxy_info.log \
+#		| stdbuf -oL grep -v web5-https \
+#		| stdbuf -oL grep -v 71.240.162.196 \
+#		| stdbuf -oL grep -v dynacomm \
+#		| stdbuf -oL haproxy_filter
+
+#else
+#	tail -f /var/log/haproxy_info.log | stdbuf -oL grep $1 | stdbuf -oL haproxy_filter
+#fi
+
+query=$1
+if [ "$query" == "" ]; then
+	tail -f /var/log/haproxy_info.log \
+		| stdbuf -oL grep -v dynacomm \
+		| stdbuf -oL haproxy_filter.sh
+else
+	tail -f /var/log/haproxy_info.log \
+		| stdbuf -oL grep -v dynacomm \
+		| stdbuf -oL grep $query \
+		| stdbuf -oL haproxy_filter.sh
+fi
+
+
+

web/haproxy/files/bin/watch_haproxy_query.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo
+echo
+echo -n "Haproxy Filter Query (blank for no filter): "; read q
+
+/usr/local/bin/watch_haproxy.sh $q

web/haproxy/files/original-debian9-haproxy-1.7.5-haproxy.cfg

@@ -0,0 +1,37 @@
+global
+	log /dev/log	local0
+	log /dev/log	local1 notice
+	chroot /var/lib/haproxy
+	stats socket /run/haproxy/admin.sock mode 660 level admin
+	stats timeout 30s
+	user haproxy
+	group haproxy
+	daemon
+
+	# Default SSL material locations
+	ca-base /etc/ssl/certs
+	crt-base /etc/ssl/private
+
+	# Default ciphers to use on SSL-enabled listening sockets.
+	# For more information, see ciphers(1SSL). This list is from:
+	#  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+	# An alternative list with additional directives can be obtained from
+	#  https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
+	ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
+	ssl-default-bind-options no-sslv3
+
+defaults
+	log	global
+	mode	http
+	option	httplog
+	option	dontlognull
+        timeout connect 5000
+        timeout client  50000
+        timeout server  50000
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 408 /etc/haproxy/errors/408.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
+	errorfile 504 /etc/haproxy/errors/504.http

web/haproxy/files/original-debian9-rsyslog-8.24.0-rsyslog.conf

@@ -0,0 +1,92 @@
+#  /etc/rsyslog.conf	Configuration file for rsyslog.
+#
+#			For more information see
+#			/usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+
+
+#################
+#### MODULES ####
+#################
+
+module(load="imuxsock") # provides support for local system logging
+module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+#module(load="imudp")
+#input(type="imudp" port="514")
+
+# provides TCP syslog reception
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf
+
+
+###############
+#### RULES ####
+###############
+
+#
+# First some standard log files.  Log by facility.
+#
+auth,authpriv.*			/var/log/auth.log
+*.*;auth,authpriv.none		-/var/log/syslog
+#cron.*				/var/log/cron.log
+daemon.*			-/var/log/daemon.log
+kern.*				-/var/log/kern.log
+lpr.*				-/var/log/lpr.log
+mail.*				-/var/log/mail.log
+user.*				-/var/log/user.log
+
+#
+# Logging for the mail system.  Split it up so that
+# it is easy to write scripts to parse these files.
+#
+mail.info			-/var/log/mail.info
+mail.warn			-/var/log/mail.warn
+mail.err			/var/log/mail.err
+
+#
+# Some "catch-all" log files.
+#
+*.=debug;\
+	auth,authpriv.none;\
+	news.none;mail.none	-/var/log/debug
+*.=info;*.=notice;*.=warn;\
+	auth,authpriv.none;\
+	cron,daemon.none;\
+	mail,news.none		-/var/log/messages
+
+#
+# Emergencies are sent to everybody logged in.
+#
+*.emerg				:omusrmsg:*

web/haproxy/files/rsyslog.conf

@@ -0,0 +1,92 @@
+#  /etc/rsyslog.conf	Configuration file for rsyslog.
+#
+#			For more information see
+#			/usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+
+
+#################
+#### MODULES ####
+#################
+
+module(load="imuxsock") # provides support for local system logging
+module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+#module(load="imudp")
+#input(type="imudp" port="514")
+
+# provides TCP syslog reception
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf
+
+
+###############
+#### RULES ####
+###############
+
+#
+# First some standard log files.  Log by facility.
+#
+auth,authpriv.*			/var/log/auth.log
+*.*;auth,authpriv.none,local0.none,local1.none      -/var/log/syslog
+#cron.*				/var/log/cron.log
+daemon.*			-/var/log/daemon.log
+kern.*				-/var/log/kern.log
+lpr.*				-/var/log/lpr.log
+mail.*				-/var/log/mail.log
+user.*				-/var/log/user.log
+
+#
+# Logging for the mail system.  Split it up so that
+# it is easy to write scripts to parse these files.
+#
+mail.info			-/var/log/mail.info
+mail.warn			-/var/log/mail.warn
+mail.err			/var/log/mail.err
+
+#
+# Some "catch-all" log files.
+#
+*.=debug;\
+	auth,authpriv.none;\
+	news.none;mail.none	-/var/log/debug
+*.=info;*.=notice;*.=warn;\
+	auth,authpriv.none;\
+	cron,daemon.none;\
+	mail,news.none,local0.none,local1.none      -/var/log/messages
+
+#
+# Emergencies are sent to everybody logged in.
+#
+*.emerg				:omusrmsg:*