summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/adei/adei.sh287
-rw-r--r--lib/adei/adei_error.sh74
-rw-r--r--lib/adei/adei_print.sh25
-rw-r--r--lib/adei/adei_version.sh51
-rw-r--r--lib/helpers.sh9
-rwxr-xr-xlib/nagios/nagios.sh125
-rw-r--r--lib/remote/README.txt34
-rw-r--r--lib/remote/print.sh168
-rw-r--r--lib/remote/report.sh51
-rw-r--r--lib/remote/run.sh13
-rw-r--r--lib/remote/status.sh196
11 files changed, 1033 insertions, 0 deletions
diff --git a/lib/adei/adei.sh b/lib/adei/adei.sh
new file mode 100644
index 0000000..3a0aeb6
--- /dev/null
+++ b/lib/adei/adei.sh
@@ -0,0 +1,287 @@
+cd $(dirname $0)
+
+MAX_SOURCE_OFFSET=3600
+MAX_MASTER_OFFSET=300
+MAX_SLAVE_OFFSET=300
+
+adei_default_timeout=${default_timeout:-120}
+
+. lib/adei/adei_version.sh
+. lib/adei/adei_print.sh
+. lib/adei/adei_error.sh
+. lib/nagios/nagios.sh
+
+shopt -s nocasematch;
+
+# Auth with redirect (-L/--location-trusted) not working. Credentials are lost on hops.
+function adei_query {
+ local resp
+ local timeout=${3:-$adei_default_timeout}
+
+ local err=0
+ if [ "$2" == "ecode" ]; then
+ url="$1&mysql=master"
+# resp=$(curl --proxy "" -f -m "$timeout" "$url" 2>&1 | grep "returned error")
+ resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" -w "%{http_code}" -o /dev/null "$url" 2>&1);
+ [ -z "$resp" -o "$resp" = "000" ] && resp="10" # error code 1
+ elif [ "$2" == "emsg" ]; then
+ url="$1&mysql=master"
+ resp=$(curl --proxy "" --location-trusted -f -m "$timeout" "$url" 2>&1 | grep -o "curl.*")
+ [ -z "$resp" ] && resp="No response from"
+ else
+ if [ "$2" == "slave" ]; then
+ url="$1&mysql=slave"
+ else
+ url="$1&mysql=master"
+ fi
+
+
+ resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" "$url"); err=$?
+ if [ $err -ne 0 ]; then
+ resp=$(adei_query "$1" "ecode" "$timeout")
+ err=$(($(($resp / 10)) + $(($resp % 10))))
+
+ resp=$(adei_query "$1" "emsg" "$timeout")
+
+ [ -n "$debug" ] && echo "$(date) $timeout Failed $url" >> /tmp/adei.log
+ else
+ [ -n "$debug" ] && echo "$(date) $timeout OK $url" >> /tmp/adei.log
+ fi
+ fi
+
+ echo -n $resp
+ return $err
+}
+
+function adei_format_query {
+ local with_auth="${1:-1}"
+ local query="$2"
+ local timeout=${3:-$adei_default_timeout}
+ local source=${4:-"$adei_source"}
+ local url="${5:-$adei_url}"
+
+ auth=""
+ [ $with_auth -gt 0 ] && auth="$adei_auth"
+ if [[ "$query" =~ \? ]]; then
+ echo "http://${auth}${url}/${query}${source}"
+ else
+ echo "http://${auth}${url}/${query}?xxxxx${source}"
+ fi
+}
+
+function adei_simple_query {
+ local req=$(adei_format_query 1 "$@")
+ adei_query "$req" "master" "$timeout"; err=$?
+ return $err
+}
+
+function adei_text_query {
+ local out # Local on the same string breaks error reporting
+ out="$(adei_simple_query "$@")"; local err=$?
+
+ if [ $err -gt 0 -o -z "$out" ]; then
+# echo "Error sending ADEI request: $(adei_format_query 0 "$@")"
+ [ $err -eq 0 ] && local err=1
+ [ -n "$out" ] && echo -n "$out"
+ return $err
+ elif [[ "$out" =~ "Error:" ]]; then
+ echo -n "$out"
+ return 7
+ else
+ echo -n "$out"
+ return 0
+ fi
+}
+
+function adei_xml_query {
+ local out # Local on the same string breaks error reporting
+ out="$(adei_simple_query "$@")"; local err=$?
+
+ if [ $err -ne 0 -o -z "$out" ]; then
+# echo "Error sending ADEI request: $(adei_format_query 0 '$@')"
+ [ $err -eq 0 ] && err=1
+ [ -n "$out" ] && echo "$out"
+ return $err
+ fi
+
+ local xml="$(echo "$out" | xmllint --format - 2>/dev/null)"
+ if [ $err -ne 0 -o -z "$out" ]; then
+ echo "$out"
+ if [[ "$out" =~ "Error:" ]]; then
+ return 7
+ else
+ return 4
+ fi
+ fi
+
+ error=$(echo "$xml" | grep "<Error>")
+ if [ -n "$error" ]; then
+ echo $error | sed -e "s|</\?Error>||g"
+ return 7
+ fi
+
+ echo "$xml"
+ return $err
+}
+
+function adei_value_query {
+ local out # Local on the same string breaks error reporting
+ out="$(adei_xml_query "$@")"; local err=$?
+ [ $err -ne 0 ] && { echo "$out"; return $err; }
+
+ local values
+ values="$(echo "$out" | grep "Value")"; err=$?
+ [ $err -ne 0 ] && return 6
+
+ echo "$values"
+}
+
+
+function adei_get_databases {
+ local out
+ out="$(adei_xml_query "list.php?target=databases" "$@")"
+ [ $err -gt 0 ] && { echo -n "$out"; return $err; }
+ echo "$out" | grep "Value" | sed -e "s/^.*db_name=\"\([^\"]*\)\".*$/\\1/" | sed -e "s/ /::space::/"
+}
+
+function adei_query_version {
+ local version # Local on the same string breaks error reporting
+
+ version="$(adei_text_query "info.php?target=version&encoding=text" "$@")"; local err=$?
+ [ $err -gt 0 ] && { echo "$version"; return $err; }
+
+ adei_version="$version"
+
+ adei_revision=$(echo $adei_version | cut -d '-' -f 1)
+ if [ "$adei_revision" == "$adei_version" ]; then
+ adei_date=""
+ else
+ adei_date=$(echo $adei_version | cut -d '-' -f 2)
+ fi
+}
+
+function adei_resolve_id {
+ fn="$1"
+ id="$2"
+ host="$3"
+ setup="$4"
+
+ local var=$(cat "$fn" | grep "^$id" | awk '{ print $2 }')
+ [ -z "$var" -a -n "$host" ] && var=$(cat "$fn" | grep "^$host" | awk '{ print $2 }')
+ [ -z "$var" -a -n "$setup" ] && var=$(cat "$fn" | grep "^$setup" | awk '{ print $2 }')
+
+ [ -n "$debug" ] && echo "$(date) resolved ($var) from $fn (id=$id, host=$host, setup=$setup) pwd ($(pwd))" >> /tmp/resolv.log
+ echo "$var"
+}
+
+function adei_init_ {
+ local id="$1" && shift # Either URL or [setup]@[host]
+# local url="$1" && shift
+ local server="$1" && shift
+ local database="$1" && [[ ! "$server" =~ \&|= ]] && shift
+ adei_args=( "$@" )
+
+ [ -z "$id" ] && { echo "ADEI ID is not specified" && exit 8 ; }
+
+
+ local url
+ local host
+ local setup
+ if [[ $id =~ http.*:// ]]; then # url
+ url="$id"
+ host="$(echo ${url#http*://} | cut -d '/' -f 1)" # parse port, maybe
+
+ unset $setup
+ elif [[ $id =~ : ]]; then # [setup@]host
+ local seho
+ IFS='@' read -ra seho <<< "$id" && shift
+ host="${seho[1]:-${seho[0]}}" # parse port, for sure
+ setup="${seho[1]:+${seho[0]}}"
+
+ unset $url
+ else # [setup]@[host] or both, no port
+ local seho
+ IFS='@' read -ra seho <<< "$id" && shift # techincally we can try to resolve to decide which is which
+
+ host="${seho[1]:-${seho[0]}}" # This could be mixed up, but it is either not important, or correct, or misconfigured anyway
+ setup="${seho[1]:+${seho[0]}}" # Only error if only setup is provided. But we handle it in resolution code which agnostic and treats them as ids
+
+ url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup")
+
+ if [ -n "$url" ]; then # Get URL, now we can determine host correctly and ignore the ids.
+ host="$(echo ${url#http*://} | cut -d '/' -f 1)"
+ unset $setup
+ else # only 'setup' is not allowed if URL is not configured, so it should be host (or error in configuration)
+ unset $url # so either we have both (correctly) or only host (correctly)
+ fi
+ fi
+
+ local hopo
+ IFS=':' read -ra hopo <<< "$host" && shift
+ host="${hopo[0]}" # only non fqdn if also url not set
+ port=":${hopo[1]:-80}"
+ [ $port = ":80" ] && port=""
+
+ local fqdn
+ if [ -z "$url" ]; then
+ url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup")
+ if [ -n "$url" ]; then
+ fqdn="$(echo ${url#http*://} | cut -d '/' -f 1)"
+
+ IFS=':' read -ra hopo <<< "$fqdn" && shift
+ host="${hopo[0]}" # again fqdn
+ port=":${hopo[1]:-80}"
+ [ $port = ":80" ] && port=""
+ else
+ fqdn=$(resolve_fqdn "$host") # this may be not fqdn
+ [ -n "$fqdn" ] && host="$fqdn"
+ url="${host}${port}/adei"
+ fi
+ fi
+
+ # In some case we miss here "setup", but surely then the password is resolved host-based
+ adei_auth=$(adei_resolve_id "security/adei.txt" "$id" "$host" "$setup")
+ [ -n "$adei_auth" ] && adei_auth="$adei_auth@"
+
+ # Now check port
+ adei_online=$(scripts/ping.pl "$host" "$port")
+ [ $adei_online -ne 1 ] && return 2
+
+ # Now build ADEI url
+
+ adei_setup="$setup"
+ adei_host="$host"
+ adei_port="$port"
+ adei_url="${url#http*://}/services"
+
+
+ adei_query_version; local err=$?
+ [ $err -gt 0 ] && return $err
+
+
+ if [[ "$server" =~ \&|= ]]; then
+ adei_source="&$server"
+ else
+ adei_source=""
+ [ -n "$adei_setup" -a "$adei_setup" != "*" ] && adei_source+="&setup=$adei_setup"
+ [ -n "$server" -a "$server" != "-" ] && adei_source+="&db_server=$server"
+
+ if [ "$database" == "#1" ]; then
+ databases="$(adei_get_databases)"; err=$?
+ [ $err -gt 0 ] && { echo "Failed to query ADEI databases: $databases"; return $err; }
+ [ -z "$databases" ] && { echo "No databases reported by ADEI"; return 6; }
+ database=$(echo "$databases" | head -n 1)
+ fi
+
+ [ -n "$database" -a "$database" != "-" ] && adei_source+="&db_name=$database"
+ fi
+
+ return 0
+}
+
+function adei_init {
+ adei_init_ "$@"; local code=$?
+
+ adei_healthy=$(($code == 0))
+ adei_process_error "$code" "" "*" "$(adei_print_status "$0" $adei_online $adei_healthy)"
+}
diff --git a/lib/adei/adei_error.sh b/lib/adei/adei_error.sh
new file mode 100644
index 0000000..9de4795
--- /dev/null
+++ b/lib/adei/adei_error.sh
@@ -0,0 +1,74 @@
+# Error codes: 1 - unknown network, 2 - unreachable, 3 - timeout, 4 - invalid XML/JSON, 5 - parsing error, 6 - set is empty, 7 - service reported error, 8 - parameter error, 9 - unknown, 10-59 - HTTP errors
+# http code (xyz) is converted to error code (xz), e.g. 404 -> 44
+
+function adei_format_http_error {
+ err=$1
+ echo -n "$(($err / 10))0$(($err % 10))"
+}
+
+# Returns 1 if printing text is recommended
+function adei_print_error {
+ local ret=0
+ local code=${1:-0}
+ local text="$2"
+
+ local service="${3#\*}"
+ service="${service:-ADEI}"
+
+ local msg
+ if [ $code -ge 30 -a $code -lt 60 ]; then
+ msg="HTTP Error: $(adei_format_http_error $code)"
+ [ $code -eq 41 ] && msg+=" (Authentication Requested)"
+ [ $code -eq 43 ] && msg+=" (Access Denied)"
+ elif [ $code -ge 20 -a $code -lt 30 ]; then
+ msg="HTTP Status: $(adei_format_http_error $code)"
+ ret=1
+ elif [ $code -eq 1 ]; then
+ msg="Error communicating with $service"
+ ret=1
+ elif [ $code -eq 2 ]; then
+ msg="ADEI host ($adei_host:$adei_port) unreachable"
+ elif [ $code -eq 3 ]; then
+ msg="Timeout waiting for $service"
+ elif [ $code -eq 4 ]; then
+ msg="Non-XML response from $service"
+ ret=1
+ elif [ $code -eq 5 ]; then
+ msg="Error parsing response from $service"
+ ret=1
+ elif [ $code -eq 6 ]; then
+ msg="Empty response from $service"
+ ret=1
+ elif [ $code -eq 7 ]; then
+ rep="$(echo "$text" | head -n 1 | sed 's/Error:\?[[:space:]]\+//i')"
+ if [ ${#rep} -lt 40 ]; then
+ msg="$service Error: $rep"
+ else
+ msg="$service returned a error"
+ ret=1
+ fi
+ elif [ $code -eq 8 ]; then
+ msg="Invalid parameter"
+ else
+ msg="Unknown Error $code while accessing $service"
+ fi
+
+ echo -n "$msg"
+ return $ret
+}
+
+function adei_process_error {
+ local code=${1:-0}; shift
+ local text="$1"; shift
+ local service="$1"; shift
+ local status="${1:-0}"
+
+ if [ $code -ne 0 ]; then
+ local msg # local on the same line break error reporting
+ msg="$(adei_print_error $code "$text" "$service")"
+ [ $? -eq 1 ] && echo "$text"
+ echo "$status $msg"
+ exit
+ fi
+}
+
diff --git a/lib/adei/adei_print.sh b/lib/adei/adei_print.sh
new file mode 100644
index 0000000..aca7735
--- /dev/null
+++ b/lib/adei/adei_print.sh
@@ -0,0 +1,25 @@
+function adei_print_status {
+ cmd="$1"
+ online="$2"
+ healthy="$3"
+
+ if [[ "$cmd" =~ check_adei_ ]]; then
+ echo "$healthy"
+ else
+ echo "$online $healthy"
+ fi
+}
+
+function format_time {
+ offset=$1
+ if [ $offset -ge 86400 ]; then
+ echo "$((offset / 86400))d"
+ elif [ $offset -ge 3600 ]; then
+ echo "$((offset / 3600))h"
+ elif [ $offset -ge 60 ]; then
+ echo "$((offset / 60))m"
+ else
+ echo "${offset}s"
+ fi
+}
+
diff --git a/lib/adei/adei_version.sh b/lib/adei/adei_version.sh
new file mode 100644
index 0000000..b71b1e1
--- /dev/null
+++ b/lib/adei/adei_version.sh
@@ -0,0 +1,51 @@
+vercomp() {
+ if [[ $1 == $2 ]]
+ then
+ return 0
+ fi
+ local IFS=.
+ local i ver1=($1) ver2=($2)
+ # fill empty fields in ver1 with zeros
+ for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
+ do
+ ver1[i]=0
+ done
+ for ((i=0; i<${#ver1[@]}; i++))
+ do
+ if [[ -z ${ver2[i]} ]]
+ then
+ # fill empty fields in ver2 with zeros
+ ver2[i]=0
+ fi
+ if ((10#${ver1[i]} > 10#${ver2[i]}))
+ then
+ return 1
+ fi
+ if ((10#${ver1[i]} < 10#${ver2[i]}))
+ then
+ return 2
+ fi
+ done
+ return 0
+}
+
+adei_version_check() {
+ vercomp $adei_revision $1
+ res=$?
+ if [ $res -eq 1 ]; then
+ return 0
+ elif [ $res -eq 2 ]; then
+ return 1;
+ fi
+
+ if [ -z "$2" -o -z "$adei_date" ]; then
+ return 0
+ fi
+
+ if [ $adei_date -ge $2 ]; then
+ return 0
+ fi
+
+ return 1
+}
+
diff --git a/lib/helpers.sh b/lib/helpers.sh
new file mode 100644
index 0000000..b562eb6
--- /dev/null
+++ b/lib/helpers.sh
@@ -0,0 +1,9 @@
+function in_array() {
+ local hay needle=$1
+ shift
+ for hay; do
+ [[ $hay == $needle ]] && return 0
+ done
+
+ return 1
+}
diff --git a/lib/nagios/nagios.sh b/lib/nagios/nagios.sh
new file mode 100755
index 0000000..40631c1
--- /dev/null
+++ b/lib/nagios/nagios.sh
@@ -0,0 +1,125 @@
+cd $(dirname "$0")
+
+. setup/parameters.sh
+
+function nagios_find_host {
+ echo "$@" | grep -oP '\-H\s*\K[^\s]*'
+}
+
+function find_host_param {
+ local id="$1" && shift
+ local pname="$1" && shift
+ local defval="$1" && shift
+
+ local val=$(cat setup/${pname}.txt | grep -P "^$id" | awk '{ print $2 }')
+ [ -n "$val" ] || val="$defval"
+
+ echo -n "$val"
+}
+
+function nagios_find_host_param {
+ local pname="$1" && shift
+ local defval="$1" && shift
+ local host=$(nagios_find_host "$@")
+
+ find_host_param "$host" "$pname" "$defval"
+}
+
+function resolve_fqdn {
+ local host="$1" && shift
+
+ if [[ ! $host =~ \. ]]; then
+ local domain=$(find_host_param "$host" domains "$default_domain")
+ host+=".$domain"
+ fi
+
+ echo -n $host
+}
+
+function resolve_ip {
+ local host="$1" && shift
+
+ if [[ ! "$host" =~ ^[[:digit:]] ]]; then
+ local fqdn=$(resolve_fqdn "$host")
+ host=$(dig +short "$fqdn" A | head -n 1 | tr -d '\n')
+ if [ $? -ne 0 -o -z "$host" ]; then
+ echo "DNS resolution for host ($fqdn) failed: $(dig +short "$fqdn" A 2>&1)"
+ exit 2
+ fi
+ fi
+
+ echo -n "$host"
+}
+
+function nagios_run {
+ if [[ "$1" =~ ^- ]]; then
+ cmd=${nagios_plugins}/$(basename $0)
+ else
+ cmd=$1 && shift
+ [[ "$cmd" =~ ^/ ]] || cmd="${nagios_plugins}/$cmd"
+ fi
+
+ ARGS=""
+ while [[ $# -gt 0 ]]; do
+ key="$1" && shift
+
+ case "$key" in
+ -H)
+ host="$1" && shift
+ ip=$(resolve_ip "$host")
+ if [ $? -eq 0 -a -n "$ip" ]; then
+ ARGS+=" ${key} $ip"
+ else
+ echo "$ip"
+ exit $?
+ fi
+ ;;
+ *)
+ ARGS+=" $key"
+ esac
+ done
+
+ [ -n "$debug" ] && echo "$(date) $cmd" "$ARGS" >> /tmp/nagios.log
+ eval "$cmd" "$ARGS"
+}
+
+function nagios_parse {
+ cmd="$1"; shift
+
+ output=$($cmd "$@")
+ exit_code=$?
+
+ if [ $exit_code -eq 0 ]; then
+ last_line=$(echo "$output" | tail -1)
+
+ status=$(echo $last_line | cut -d ' ' -f 1)
+ [[ "$status" =~ ^[0-9]$ ]] || status=0
+ echo -n ${last_line#$status}
+ else
+ status=0
+ fi
+
+ if [ $status -ne 1 ]; then
+ echo -n " -- "
+
+ lines=$(echo "$output" | sed '/^[[:space:]]*$/d' | wc -l)
+ if [ $lines -gt 1 ]; then
+ echo "$output" | sed '/^[[:space:]]*$/d' | head -n 1 | tr -d '\n'
+ echo " ... "
+ echo "------------------"
+ echo "$output" | head -n -1
+ else
+ echo "$output" | sed '/^[[:space:]]*$/d' | head -n -1
+ fi
+ else
+ echo
+ fi
+
+ if [ "$status" -eq 0 ]; then
+ exit "2"
+ elif [ "$status" -eq 1 ]; then
+ exit "0"
+ else
+ exit "1"
+ fi
+}
diff --git a/lib/remote/README.txt b/lib/remote/README.txt
new file mode 100644
index 0000000..c5998f8
--- /dev/null
+++ b/lib/remote/README.txt
@@ -0,0 +1,34 @@
+conky format
+============
+ - Server status
+ * ping works
+ * no output from 'check_server_status.sh' and additional listed checks
+ [*] Checks for additional listed services 'check_<service>_status.sh'.
+ - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string).
+ - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow.
+ [headers]
+
+ ... Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format
+ server:port[:service] text....
+
+ - Extra info for DarkSoft servers
+ * VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here.
+ # # Traffic on VPN interface from/to
+
+ - Server traffic
+ in/out/forw. Yellow/Red if above specified threshold. No additional errors are reported here.
+
+ - Services (operated by local scripts in 'services' directory)
+ * Online
+ * Healthy
+ [headers] Information about service operation
+ [errors] Short information on critical errors are reported here (in conky, we push everything in the text section)
+
+ ... Larger textual information about the problems.
+
+
+ Service format is following (status is on the last line and mandatory):
+ * Critical Lines starting with '*' contain critical errors and highlighted (remote) / reported in the service table (conky)
+ Text This is normal problem description reported in the textual area
+ # # [header] Two statuses (0 - error, 1 - ok, 2 - warning) and optional header with information about service operation
+ \ No newline at end of file
diff --git a/lib/remote/print.sh b/lib/remote/print.sh
new file mode 100644
index 0000000..a0fdcbe
--- /dev/null
+++ b/lib/remote/print.sh
@@ -0,0 +1,168 @@
+# underlines 4;30,... blinking 5:30,...
+# backgrounds: 40,...
+
+declare -A colors=(
+ [black]='0;30'
+ [red]='0;31'
+ [green]='0;32'
+ [orange]='0;33'
+ [blue]='0;34'
+ [purple]='0;35'
+ [cyan]='0;36'
+ [lightgray]='0;37'
+ [gray]='1;30'
+ [lightred]='1;31'
+ [lightgreen]='1;32'
+ [yellow]='1;33'
+ [lightblue]='1;34'
+ [lightpurple]='1;35'
+ [lightcyan]='1;36'
+ [white]='1;37'
+)
+
+background=""
+foreground=""
+
+
+
+
+# Fixing color map
+#echo -ne "\e[44m"
+#echo -ne '\x1B[K'
+
+function set_background {
+ if [ -n "$background" ]; then
+ echo -ne "$background"
+ echo -ne '\x1B[K'
+ fi
+}
+
+function set_color {
+ local color=${1:-""}
+ local effect=${2:-""}
+
+ c=${colors[$color]}
+
+
+ case "$effect" in
+ 'u')
+ c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/")
+ ;;
+ esac
+
+ [ -n "$color" ] && echo -ne "\033[${c}m"
+ set_background
+}
+
+function reset_color {
+ echo -ne "\033[0m"
+
+ set_background
+ [ -n "$foreground" ] && set_color "$foreground"
+}
+
+function finish {
+ echo -ne "\033[0m"
+ echo
+}
+
+function configure_palete {
+ case $1 in
+ blue)
+ background="\e[1;44m"
+ foreground="white"
+ ;;
+ *)
+ # Lightgray and Yellow are almost invisible on white
+ colors[yellow]=${colors[orange]}
+ colors[white]=${colors[black]}
+ colors[lightgray]=${colors[gray]}
+ colors[cyan]=${colors[blue]}
+ ;;
+ esac
+
+ set_background
+ reset_color
+}
+
+function decorate {
+ local output="$1"
+ local color=${2:-""}
+ local effect=${3:-""}
+
+ if [ -n "$color" ]; then
+ set_color $color $effect
+ echo -ne "${output}"
+ reset_color
+ else
+ echo -ne "$output"
+ fi
+}
+
+function eol {
+ sed 's/$/\x1B[K/'
+}
+
+function print_eol {
+ echo -e '\x1B[K'
+}
+
+
+
+function print {
+ local output="$1"
+ local color=${2:-""}
+ local char_limit=${3:-$fold}
+ local line_limit=${4:-0}
+
+ local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit"
+ [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit"
+
+ eval "$cmd" | eol
+}
+
+
+function print_hline {
+ local color=${1:-"gray"}
+ local cols=${COLUMNS:-$(tput cols)}
+
+ decorate $(printf '%*s' $cols '' | tr ' ' -) "$color"
+ echo | eol
+}
+
+function print_table {
+ # The problem here is that all escapes for colors are treated as normal visible symbols
+ sed "s/ ::: /@/g" | column -t -s "@" -o " " | eol
+}
+
+function decorate_table {
+ print_hline
+ cat -
+ print_hline
+}
+
+function print_status {
+ local status="$1"
+
+ if [[ $status =~ ^.$ ]]; then
+ case $status in
+ 0)
+ decorate "*" "red"
+ ;;
+ 1)
+ decorate "*" "green"
+ ;;
+ 2)
+ decorate "*" "yellow"
+ ;;
+ *)
+ decorate "$status" "red"
+ ;;
+ esac
+ else
+ decorate "?" "red"
+ fi
+ echo -n " "
+}
+
+clear
diff --git a/lib/remote/report.sh b/lib/remote/report.sh
new file mode 100644
index 0000000..e6766a4
--- /dev/null
+++ b/lib/remote/report.sh
@@ -0,0 +1,51 @@
+function simple_report {
+ local generator="$1"
+ local filter="${2:-cat}"
+ {
+ eval "$generator 12>/dev/fd/12 | $filter"
+ report=$(cat<&12)
+ } 12<<EOF
+EOF
+ wait
+}
+
+
+function short_report {
+ local generator="$1"
+ local filter="${2:-print_table | decorate_table}"
+
+ configure_palete "$palete"
+ simple_report "$generator" "$filter"
+ finish
+}
+
+
+function standart_report {
+ local generator="$1"
+ local filter="${2:-print_table | decorate_table}"
+
+ configure_palete "$palete"
+ simple_report "$generator" "$filter"
+
+ print "$report"
+ echo
+ finish
+}
+
+
+function report {
+ # Eval breaks output for some reason
+ case $report in
+ simple_report)
+ simple_report "$@"
+ ;;
+ short_report)
+ short_report "$@"
+ ;;
+ standart_report)
+ standart_report "$@"
+ ;;
+ default)
+ echo "Report is not implemented"
+ esac
+}
diff --git a/lib/remote/run.sh b/lib/remote/run.sh
new file mode 100644
index 0000000..fdd7311
--- /dev/null
+++ b/lib/remote/run.sh
@@ -0,0 +1,13 @@
+function run_ {
+ local output=$(eval source "$@")
+
+ flock -x $0 echo -e "$output"
+}
+
+function run {
+ if [ $parallel -gt 0 ]; then
+ run_ "$@" &
+ else
+ run_ "$@"
+ fi
+}
diff --git a/lib/remote/status.sh b/lib/remote/status.sh
new file mode 100644
index 0000000..7bef6c8
--- /dev/null
+++ b/lib/remote/status.sh
@@ -0,0 +1,196 @@
+. setup/remote.sh
+. lib/remote/print.sh
+. lib/remote/report.sh
+. lib/remote/run.sh
+
+function check_server_traffic {
+ local result="$1"
+ local yellow=${2:-0}
+ local red=${3:-0}
+
+ local name=("in" "out" "fwd")
+
+ local traf
+ read -ra traf <<< $result
+ if [ ${#traf[@]} -ne 3 ]; then
+ [ -z "$result" ] && result="No info received"
+ print "$result" "red" $fold 1
+ return
+ fi
+
+ local output=""
+ for i in $(seq 0 2); do
+ if ! [[ ${traf[$i]} =~ ^[0-9]+$ ]]; then
+ print "$result" "red" $fold 1
+ return
+ fi
+
+ local val=$(printf "% 4u" $((${traf[$i]} / 1024 / 1024 / 1024)))
+ if [ $red -gt 0 -a $val -gt $red ]; then
+ val=$(decorate "$val GB" "red")
+ elif [ $yellow -gt 0 -a $val -gt $yellow ]; then
+ val=$(decorate "$val GB" "yellow")
+ else
+ val="$val GB"
+ fi
+
+# output="$output${output:+, }${name[$i]}: $val"
+ output="$output${output:+, } $val"
+ done
+
+# print "$output"
+ print "in/out/fwd: $output"
+}
+
+function check_server_ {
+ local service=$1 && shift
+
+ local hopo
+ IFS=':' read -ra hopo <<< $1 && shift
+ local host=${hopo[0]}
+ local port=${hopo[1]:-22}
+
+ local result=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_$service.sh 2>&1 )
+
+ case "$service" in
+ 'traffic')
+ check_server_traffic "$result" "$@"
+ ;;
+ *)
+ print "$result"
+ esac
+
+}
+
+
+function check_server_status {
+ local hopo
+ IFS=':' read -ra hopo <<< $1 && shift
+ local host=${hopo[0]}
+ local port=${hopo[1]:-22}
+ local services="$@"
+
+ local ping_output=$(scripts/ping.pl $host $port $timeout)
+ print_status "$ping_output"
+
+ if [ "$services" == "-" ]; then return; fi
+
+ if [ "$ping_output" == "1" ]; then
+ local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1)
+
+ local status=1
+ [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n"
+ print_status $status
+
+ # Check Additional services, error is reported if anything provides information
+ local headers=""
+ if [ -n "$services" ]; then
+ for service in "$services"; do
+ if [[ "$service" =~ ^vpn/(.*)$ ]]; then
+ local ip=${BASH_REMATCH[1]}
+ [ $(hostname) == "styx" ] && check_server_status "$ip" "-"
+ else
+ local service_status=1
+ local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d')
+ if [ -n "$service_output" ]; then
+ first_line=$(echo "$service_output" | head -n 1)
+ if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then
+ service_status=${BASH_REMATCH[1]}
+ service_output=$(sed "1 d" <<< "$service_output")
+ if [ -n "${BASH_REMATCH[2]}" ]; then
+ service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}") #"
+ headers="$headers $service_header"
+ fi
+ else
+ service_status=0
+ fi
+
+ if [ -n "$service_output" ]; then
+ [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")"
+ output="${output}\n $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/ /')"
+ fi
+ fi
+ fi
+
+ done
+ fi
+
+ print " ::: ${headers}"
+ #report="$report<section>$output"
+ if [ -n "$output" ]; then
+ flock -x $0 echo "${output}" >&12
+ fi
+ else
+ print_status "x"
+ for service in "$services"; do
+ print_status "x"
+ done
+ fi
+}
+
+function check_service {
+ local service="$1" && shift
+ local id="$1" && shift
+ local args="$@"
+
+ local output=$(service/check_${service}.sh ${id} "$@" 2>&1 | sed 's/^[[:space:]]*//;/^$/d')
+
+ local info=$(echo "$output" | tail -n 1)
+
+ local online=0
+ local health=0
+ local header=""
+ if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then
+ online=${BASH_REMATCH[1]}
+ health=${BASH_REMATCH[2]}
+ header=${BASH_REMATCH[3]}
+
+ output=$(sed '$ d' <<< "$output")
+ fi
+
+ print_status "$online"
+ print_status "$health"
+
+ header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header") # "
+ print " ::: ${header}"
+
+
+ important=$(grep "^\*" <<< "$output")
+ messages=$(grep -v "^\*" <<< "$output")
+
+ if [ -n "$output" ]; then
+ output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n"
+ flock -x $0 echo "${output}" >&12
+ fi
+}
+
+
+function check__ {
+ local args
+ local title="$1" && shift
+ read -ra args <<< "$1" && shift
+ local host=${args[0]};
+
+ print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; "
+
+ local service
+ for service in "$@"; do
+ service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service") # "
+ ( eval "$service" )
+ done
+ print_eol
+}
+
+function check_ {
+ # Buffer the output
+ local output=$(check__ "$@")
+ echo -e "$output"
+}
+
+function check {
+ if [ $parallel -gt 0 ]; then
+ check_ "$@" &
+ else
+ check_ "$@"
+ fi
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-08 16:41:39 +0000