Integrate also nagios-based monitoring here

11 files changed, 1033 insertions, 0 deletions

diff --git a/lib/adei/adei.sh b/lib/adei/adei.sh
new file mode 100644
index 0000000..3a0aeb6
--- /dev/null
+++ b/lib/adei/adei.sh
@@ -0,0 +1,287 @@
+cd $(dirname $0)
+
+MAX_SOURCE_OFFSET=3600
+MAX_MASTER_OFFSET=300
+MAX_SLAVE_OFFSET=300
+
+adei_default_timeout=${default_timeout:-120}
+
+. lib/adei/adei_version.sh
+. lib/adei/adei_print.sh
+. lib/adei/adei_error.sh
+. lib/nagios/nagios.sh
+
+shopt -s nocasematch;
+
+# Auth with redirect (-L/--location-trusted) not working. Credentials are lost on hops.
+function adei_query {
+    local resp
+    local timeout=${3:-$adei_default_timeout}
+
+    local err=0
+    if [ "$2" == "ecode" ]; then
+        url="$1&mysql=master"
+#	resp=$(curl --proxy "" -f -m "$timeout" "$url" 2>&1 | grep "returned error")
+	resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" -w "%{http_code}" -o /dev/null "$url" 2>&1);
+	[ -z "$resp" -o "$resp" = "000" ] && resp="10" # error code 1
+    elif [ "$2" == "emsg" ]; then
+        url="$1&mysql=master"
+	resp=$(curl --proxy "" --location-trusted -f -m "$timeout" "$url" 2>&1 | grep -o "curl.*")
+	[ -z "$resp" ] && resp="No response from"
+    else
+        if [ "$2" == "slave" ]; then
+            url="$1&mysql=slave"
+        else
+            url="$1&mysql=master"
+        fi
+
+
+	resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" "$url"); err=$?
+	if [ $err -ne 0 ]; then
+	    resp=$(adei_query "$1" "ecode" "$timeout")
+	    err=$(($(($resp / 10)) + $(($resp % 10))))
+
+	    resp=$(adei_query "$1" "emsg" "$timeout")
+
+	    [ -n "$debug" ] && echo "$(date) $timeout Failed $url" >> /tmp/adei.log
+	else
+	    [ -n "$debug" ] && echo "$(date) $timeout OK     $url" >> /tmp/adei.log
+	fi
+    fi
+
+    echo -n $resp
+    return $err
+}
+
+function adei_format_query {
+    local with_auth="${1:-1}"
+    local query="$2"
+    local timeout=${3:-$adei_default_timeout}
+    local source=${4:-"$adei_source"}
+    local url="${5:-$adei_url}"
+
+    auth=""
+    [ $with_auth -gt 0 ] && auth="$adei_auth"
+    if [[ "$query" =~ \? ]]; then
+	echo "http://${auth}${url}/${query}${source}"
+    else
+	echo "http://${auth}${url}/${query}?xxxxx${source}"
+    fi
+}
+
+function adei_simple_query {
+    local req=$(adei_format_query 1 "$@")
+    adei_query "$req" "master" "$timeout"; err=$?
+    return $err
+}
+
+function adei_text_query {
+    local out	# Local on the same string breaks error reporting
+    out="$(adei_simple_query "$@")"; local err=$?
+
+    if [ $err -gt 0 -o -z "$out" ]; then
+#	echo "Error sending ADEI request: $(adei_format_query 0 "$@")"
+	[ $err -eq 0 ] && local err=1
+	[ -n "$out" ] && echo -n "$out"
+	return $err
+    elif [[ "$out" =~ "Error:" ]]; then
+	echo -n "$out"
+	return 7
+    else
+	echo -n "$out"
+	return 0
+    fi
+}
+
+function adei_xml_query {
+    local out	# Local on the same string breaks error reporting
+    out="$(adei_simple_query "$@")"; local err=$?
+
+    if [ $err -ne 0 -o -z "$out" ]; then
+#	echo "Error sending ADEI request: $(adei_format_query 0 '$@')"
+	[ $err -eq 0 ] && err=1
+	[ -n "$out" ] && echo "$out"
+	return $err
+    fi
+
+    local xml="$(echo "$out" | xmllint --format - 2>/dev/null)"
+    if [ $err -ne 0 -o -z "$out" ]; then
+	echo "$out"
+	if [[ "$out" =~ "Error:" ]]; then
+	    return 7
+	else
+	    return 4
+	fi
+    fi
+
+    error=$(echo "$xml" | grep "<Error>")
+    if [ -n "$error" ]; then
+	echo $error | sed -e "s|</\?Error>||g"
+	return 7
+    fi
+
+    echo "$xml"
+    return $err
+}
+
+function adei_value_query {
+    local out	# Local on the same string breaks error reporting
+    out="$(adei_xml_query "$@")"; local err=$?
+    [ $err -ne 0 ] && { echo "$out"; return $err; }
+
+    local values
+    values="$(echo "$out" | grep "Value")"; err=$?
+    [ $err -ne 0 ] && return 6
+
+    echo "$values"
+}
+
+
+function adei_get_databases {
+    local out
+    out="$(adei_xml_query "list.php?target=databases" "$@")"
+    [ $err -gt 0 ] && { echo -n "$out"; return $err; }
+    echo "$out" | grep "Value" | sed -e "s/^.*db_name=\"\([^\"]*\)\".*$/\\1/" | sed -e "s/ /::space::/"
+}
+
+function adei_query_version {
+    local version	# Local on the same string breaks error reporting
+
+    version="$(adei_text_query "info.php?target=version&encoding=text" "$@")"; local err=$?
+    [ $err -gt 0 ] && { echo "$version"; return $err; }
+
+    adei_version="$version"
+
+    adei_revision=$(echo $adei_version | cut -d '-' -f 1)
+    if [ "$adei_revision" == "$adei_version" ]; then 
+	adei_date=""
+    else
+	adei_date=$(echo $adei_version | cut -d '-' -f 2)
+    fi
+}
+
+function adei_resolve_id {
+    fn="$1"
+    id="$2"
+    host="$3"
+    setup="$4"
+
+    local var=$(cat "$fn"  | grep "^$id" | awk '{ print $2 }')
+    [ -z "$var" -a -n "$host" ] && var=$(cat "$fn"  | grep "^$host"  | awk '{ print $2 }')
+    [ -z "$var" -a -n "$setup" ] && var=$(cat "$fn"  | grep "^$setup" | awk '{ print $2 }')
+
+    [ -n "$debug" ] && echo "$(date) resolved ($var) from $fn (id=$id, host=$host, setup=$setup) pwd ($(pwd))" >> /tmp/resolv.log
+    echo "$var"
+}
+
+function adei_init_ {
+    local id="$1" && shift		# Either URL or [setup]@[host]
+#    local url="$1" && shift
+    local server="$1" && shift
+    local database="$1" && [[ ! "$server" =~ \&|= ]] && shift
+    adei_args=( "$@" )
+
+    [ -z "$id" ] && { echo "ADEI ID is not specified" && exit 8 ; }
+
+
+    local url
+    local host
+    local setup
+    if [[ $id =~ http.*:// ]]; then				# url
+	url="$id"
+	host="$(echo ${url#http*://} | cut -d '/' -f 1)"	# parse port, maybe
+
+	unset $setup
+    elif [[ $id =~ : ]]; then					# [setup@]host
+	local seho
+	IFS='@' read -ra seho <<< "$id" && shift
+	host="${seho[1]:-${seho[0]}}"				# parse port, for sure
+        setup="${seho[1]:+${seho[0]}}"
+
+        unset $url
+    else							# [setup]@[host] or both, no port
+	local seho
+	IFS='@' read -ra seho <<< "$id" && shift		# techincally we can try to resolve to decide which is which
+
+	host="${seho[1]:-${seho[0]}}"				# This could be mixed up, but it is either not important, or correct, or misconfigured anyway
+        setup="${seho[1]:+${seho[0]}}"				# Only error if only setup is provided. But we handle it in resolution code which agnostic and treats them as ids 
+
+        url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup")
+
+        if [ -n "$url" ]; then					# Get URL, now we can determine host correctly and ignore the ids.
+	    host="$(echo ${url#http*://} | cut -d '/' -f 1)"
+	    unset $setup
+	else							# only 'setup' is not allowed if URL is not configured, so it should be host (or error in configuration)
+	    unset $url						# so either we have both (correctly) or only host (correctly)
+	fi
+    fi
+
+    local hopo
+    IFS=':' read -ra hopo <<< "$host" && shift
+    host="${hopo[0]}"					# only non fqdn if also url not set
+    port=":${hopo[1]:-80}"
+    [ $port = ":80" ] && port=""
+
+    local fqdn
+    if [ -z "$url" ]; then
+        url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup")
+	if [ -n "$url" ]; then
+	    fqdn="$(echo ${url#http*://} | cut -d '/' -f 1)"
+
+	    IFS=':' read -ra hopo <<< "$fqdn" && shift
+	    host="${hopo[0]}"				# again fqdn
+	    port=":${hopo[1]:-80}"
+	    [ $port = ":80" ] && port=""
+	else
+	    fqdn=$(resolve_fqdn "$host")		# this may be not fqdn
+	    [ -n "$fqdn" ] && host="$fqdn"
+	    url="${host}${port}/adei"
+	fi
+    fi
+
+    # In some case we miss here "setup", but surely then the password is resolved host-based
+    adei_auth=$(adei_resolve_id "security/adei.txt" "$id" "$host" "$setup")
+    [ -n "$adei_auth" ] && adei_auth="$adei_auth@"
+
+    # Now check port
+    adei_online=$(scripts/ping.pl "$host" "$port")
+    [ $adei_online -ne 1 ] && return 2
+    
+    # Now build ADEI url
+
+    adei_setup="$setup"
+    adei_host="$host"
+    adei_port="$port"
+    adei_url="${url#http*://}/services"
+
+
+    adei_query_version; local err=$?
+    [ $err -gt 0 ] && return $err
+
+
+    if [[ "$server" =~ \&|= ]]; then
+	adei_source="&$server"
+    else
+	adei_source=""
+	[ -n "$adei_setup" -a "$adei_setup" != "*" ] && adei_source+="&setup=$adei_setup"
+        [ -n "$server" -a "$server" != "-" ] && adei_source+="&db_server=$server"
+
+	if [ "$database" == "#1" ]; then
+	    databases="$(adei_get_databases)"; err=$?
+	    [ $err -gt 0 ] && { echo "Failed to query ADEI databases: $databases"; return $err; }
+	    [ -z "$databases" ] && { echo "No databases reported by ADEI"; return 6; }
+	    database=$(echo "$databases" | head -n 1)
+	fi
+
+	[ -n "$database" -a "$database" != "-" ] && adei_source+="&db_name=$database"
+    fi
+
+    return 0
+}
+
+function adei_init {
+    adei_init_ "$@"; local code=$?
+
+    adei_healthy=$(($code == 0))
+    adei_process_error "$code" "" "*" "$(adei_print_status "$0" $adei_online $adei_healthy)"
+}
diff --git a/lib/adei/adei_error.sh b/lib/adei/adei_error.sh
new file mode 100644
index 0000000..9de4795
--- /dev/null
+++ b/lib/adei/adei_error.sh
@@ -0,0 +1,74 @@
+# Error codes: 1 - unknown network, 2 - unreachable, 3 - timeout, 4 - invalid XML/JSON, 5 - parsing error, 6 - set is empty, 7 - service reported error, 8 - parameter error, 9 - unknown, 10-59 - HTTP errors
+# http code (xyz) is converted to error code (xz), e.g. 404 -> 44
+
+function adei_format_http_error {
+    err=$1
+    echo -n "$(($err / 10))0$(($err % 10))"
+}
+
+# Returns 1 if printing text is recommended
+function adei_print_error {
+    local ret=0
+    local code=${1:-0}
+    local text="$2"
+
+    local service="${3#\*}"
+    service="${service:-ADEI}"
+
+    local msg
+    if [ $code -ge 30 -a $code -lt 60 ]; then
+	msg="HTTP Error: $(adei_format_http_error $code)"
+	[ $code -eq 41 ] && msg+=" (Authentication Requested)"
+	[ $code -eq 43 ] && msg+=" (Access Denied)"
+    elif [ $code -ge 20 -a $code -lt 30 ]; then
+	msg="HTTP Status: $(adei_format_http_error $code)"
+	ret=1
+    elif [ $code -eq 1 ]; then
+	msg="Error communicating with $service"
+	ret=1
+    elif [ $code -eq 2 ]; then
+	msg="ADEI host ($adei_host:$adei_port) unreachable"
+    elif [ $code -eq 3 ]; then
+	msg="Timeout waiting for $service"
+    elif [ $code -eq 4 ]; then
+	msg="Non-XML response from $service"
+	ret=1
+    elif [ $code -eq 5 ]; then 
+	msg="Error parsing response from $service"
+	ret=1
+    elif [ $code -eq 6 ]; then 
+	msg="Empty response from $service"
+	ret=1
+    elif [ $code -eq 7 ]; then
+	rep="$(echo "$text" | head -n 1 | sed 's/Error:\?[[:space:]]\+//i')"
+	if [ ${#rep} -lt 40 ]; then
+	    msg="$service Error: $rep"
+	else
+	    msg="$service returned a error"
+	    ret=1
+	fi
+    elif [ $code -eq 8 ]; then 
+	msg="Invalid parameter"
+    else
+	msg="Unknown Error $code while accessing $service"
+    fi
+
+    echo -n "$msg"
+    return $ret
+}
+
+function adei_process_error {
+    local code=${1:-0}; shift
+    local text="$1"; shift
+    local service="$1"; shift
+    local status="${1:-0}"
+
+    if [ $code -ne 0 ]; then
+	local msg	# local on the same line break error reporting
+	msg="$(adei_print_error $code "$text" "$service")"
+	[ $? -eq 1 ] && echo "$text"
+	echo "$status $msg"
+	exit
+    fi
+}
+
diff --git a/lib/adei/adei_print.sh b/lib/adei/adei_print.sh
new file mode 100644
index 0000000..aca7735
--- /dev/null
+++ b/lib/adei/adei_print.sh
@@ -0,0 +1,25 @@
+function adei_print_status {
+    cmd="$1"
+    online="$2"
+    healthy="$3"
+
+    if [[ "$cmd" =~ check_adei_ ]]; then
+	echo "$healthy"
+    else
+	echo "$online $healthy"
+    fi
+}
+
+function format_time {
+    offset=$1
+    if [ $offset -ge 86400 ]; then
+        echo "$((offset / 86400))d"
+    elif [ $offset -ge 3600 ]; then
+        echo "$((offset / 3600))h"
+    elif [ $offset -ge 60 ]; then
+        echo "$((offset / 60))m"
+    else
+        echo "${offset}s"
+    fi
+}
+
diff --git a/lib/adei/adei_version.sh b/lib/adei/adei_version.sh
new file mode 100644
index 0000000..b71b1e1
--- /dev/null
+++ b/lib/adei/adei_version.sh
@@ -0,0 +1,51 @@
+vercomp() {
+    if [[ $1 == $2 ]]
+    then
+        return 0
+    fi
+    local IFS=.
+    local i ver1=($1) ver2=($2)
+    # fill empty fields in ver1 with zeros
+    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
+    do
+        ver1[i]=0
+    done
+    for ((i=0; i<${#ver1[@]}; i++))
+    do
+        if [[ -z ${ver2[i]} ]]
+        then
+            # fill empty fields in ver2 with zeros
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]}))
+        then
+            return 1
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]}))
+        then
+            return 2
+        fi
+    done
+    return 0
+}
+
+adei_version_check() {
+    vercomp $adei_revision $1
+    res=$?
+    if [ $res -eq 1 ]; then 
+	return 0
+    elif [ $res -eq 2 ]; then 
+	return 1;
+    fi
+    
+    if [ -z "$2" -o -z "$adei_date" ]; then 
+	return 0
+    fi
+
+    if [ $adei_date -ge $2 ]; then
+	return 0
+    fi
+    
+    return 1
+}
+
diff --git a/lib/helpers.sh b/lib/helpers.sh
new file mode 100644
index 0000000..b562eb6
--- /dev/null
+++ b/lib/helpers.sh
@@ -0,0 +1,9 @@
+function in_array() {
+    local hay needle=$1
+    shift
+    for hay; do
+        [[ $hay == $needle ]] && return 0
+    done
+
+    return 1
+}
diff --git a/lib/nagios/nagios.sh b/lib/nagios/nagios.sh
new file mode 100755
index 0000000..40631c1
--- /dev/null
+++ b/lib/nagios/nagios.sh
@@ -0,0 +1,125 @@
+cd $(dirname "$0")
+
+. setup/parameters.sh
+
+function nagios_find_host {
+    echo "$@" | grep -oP '\-H\s*\K[^\s]*'
+}
+
+function find_host_param {
+    local id="$1" && shift
+    local pname="$1" && shift
+    local defval="$1" && shift
+
+    local val=$(cat setup/${pname}.txt  | grep -P "^$id" | awk '{ print $2 }')
+    [ -n "$val" ] || val="$defval"
+
+    echo -n "$val"
+}
+
+function nagios_find_host_param {
+    local pname="$1" && shift
+    local defval="$1" && shift
+    local host=$(nagios_find_host "$@")
+
+    find_host_param "$host" "$pname" "$defval"
+}
+
+function resolve_fqdn {
+    local host="$1" && shift
+
+    if [[ ! $host =~ \. ]]; then
+	local domain=$(find_host_param "$host" domains "$default_domain")
+	host+=".$domain"
+    fi
+
+    echo -n $host
+}
+
+function resolve_ip {
+    local host="$1" && shift
+
+    if [[ ! "$host" =~ ^[[:digit:]] ]]; then
+	local fqdn=$(resolve_fqdn "$host")
+	host=$(dig +short "$fqdn" A | head -n 1 | tr -d '\n')
+	if [ $? -ne 0 -o -z "$host" ]; then
+	    echo "DNS resolution for host ($fqdn) failed: $(dig +short "$fqdn" A 2>&1)"
+	    exit 2
+	fi
+    fi
+
+    echo -n "$host"
+}
+
+function nagios_run {
+    if [[ "$1" =~ ^- ]]; then
+	cmd=${nagios_plugins}/$(basename $0)
+    else
+	cmd=$1 && shift
+	[[ "$cmd" =~ ^/ ]] || cmd="${nagios_plugins}/$cmd"
+    fi
+
+    ARGS=""
+    while [[ $# -gt 0 ]]; do
+	key="$1" && shift
+    
+	case "$key" in 
+          -H)
+	    host="$1" && shift
+	    ip=$(resolve_ip "$host")
+	    if [ $? -eq 0 -a -n "$ip" ]; then
+		ARGS+=" ${key} $ip"
+	    else
+		echo "$ip"
+		exit $?
+	    fi
+	    ;;
+          *)
+	    ARGS+=" $key"
+	esac
+    done
+
+    [ -n "$debug" ] && echo "$(date) $cmd" "$ARGS" >> /tmp/nagios.log
+    eval "$cmd" "$ARGS"
+}
+
+function nagios_parse {
+    cmd="$1"; shift
+
+    output=$($cmd "$@")
+    exit_code=$?
+
+    if [ $exit_code -eq 0 ]; then
+	last_line=$(echo "$output" | tail -1)
+
+	status=$(echo $last_line | cut -d ' ' -f 1)
+	[[ "$status" =~ ^[0-9]$ ]] || status=0
+	echo -n ${last_line#$status}
+    else 
+	status=0
+    fi
+
+    if [ $status -ne 1 ]; then 
+	echo -n " -- "
+
+	lines=$(echo "$output" | sed '/^[[:space:]]*$/d' | wc -l)
+	if [ $lines -gt 1 ]; then
+	    echo "$output" | sed '/^[[:space:]]*$/d' | head -n 1 | tr -d '\n'
+	    echo " ... "
+	    echo "------------------"
+	    echo "$output" | head -n -1
+	else
+	    echo "$output" | sed '/^[[:space:]]*$/d' | head -n -1
+	fi
+    else
+	echo
+    fi
+
+    if [ "$status" -eq 0 ]; then
+	exit "2"
+    elif [ "$status" -eq 1 ]; then
+	exit "0"
+    else
+	exit "1"
+    fi
+}
diff --git a/lib/remote/README.txt b/lib/remote/README.txt
new file mode 100644
index 0000000..c5998f8
--- /dev/null
+++ b/lib/remote/README.txt
@@ -0,0 +1,34 @@
+conky format
+============
+ - Server status
+    *                   ping works
+    *                   no output from 'check_server_status.sh' and additional listed checks 
+    [*]                 Checks for additional listed services 'check_<service>_status.sh'. 
+                        - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string).
+                        - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow.
+    [headers]             
+                        
+    ...                 Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format
+                        server:port[:service] text....
+    
+ - Extra info for DarkSoft servers
+   *                    VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here.
+        # #             Traffic on VPN interface from/to
+
+ - Server traffic
+    in/out/forw.        Yellow/Red if above specified threshold. No additional errors are reported here.
+
+ - Services (operated by local scripts in 'services' directory)
+    *                   Online  
+    *                   Healthy
+    [headers]           Information about service operation
+        [errors]        Short information on critical errors are reported here (in conky, we push everything in the text section)
+        
+    ...                 Larger textual information about the problems.
+
+
+    Service format is following (status is on the last line and mandatory):
+        * Critical      Lines starting with '*' contain critical errors and highlighted (remote) / reported in the service table (conky)
+        Text            This is normal problem description reported in the textual area
+        # # [header]    Two statuses (0 - error, 1 - ok, 2 - warning) and optional header with information about service operation
+    
\ No newline at end of file
diff --git a/lib/remote/print.sh b/lib/remote/print.sh
new file mode 100644
index 0000000..a0fdcbe
--- /dev/null
+++ b/lib/remote/print.sh
@@ -0,0 +1,168 @@
+# underlines    4;30,...        blinking 5:30,...
+# backgrounds:  40,...
+
+declare -A colors=(
+    [black]='0;30'
+    [red]='0;31'
+    [green]='0;32'
+    [orange]='0;33'
+    [blue]='0;34'
+    [purple]='0;35'
+    [cyan]='0;36'
+    [lightgray]='0;37'
+    [gray]='1;30'
+    [lightred]='1;31'
+    [lightgreen]='1;32'
+    [yellow]='1;33'
+    [lightblue]='1;34'
+    [lightpurple]='1;35'
+    [lightcyan]='1;36'
+    [white]='1;37'
+)
+
+background=""
+foreground=""
+
+
+
+
+# Fixing color map
+#echo -ne "\e[44m"
+#echo -ne '\x1B[K'
+
+function set_background {
+    if [ -n "$background" ]; then
+        echo -ne "$background"
+        echo -ne '\x1B[K'
+    fi
+}
+
+function set_color {
+    local color=${1:-""}
+    local effect=${2:-""}
+    
+    c=${colors[$color]} 
+
+    
+    case "$effect" in
+      'u')
+        c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/")
+        ;;
+    esac
+    
+    [ -n "$color" ] && echo -ne "\033[${c}m"
+    set_background
+}
+
+function reset_color {
+    echo -ne "\033[0m"
+
+    set_background
+    [ -n "$foreground" ] && set_color "$foreground"
+}
+
+function finish {
+    echo -ne "\033[0m"
+    echo
+}
+
+function configure_palete {
+    case $1 in
+      blue)
+        background="\e[1;44m"
+        foreground="white"
+        ;;
+      *)
+        # Lightgray and Yellow are almost invisible on white
+        colors[yellow]=${colors[orange]}
+        colors[white]=${colors[black]}
+        colors[lightgray]=${colors[gray]}
+        colors[cyan]=${colors[blue]}
+        ;;
+    esac
+    
+    set_background
+    reset_color
+}
+
+function decorate {
+    local output="$1"
+    local color=${2:-""}
+    local effect=${3:-""}
+    
+    if [ -n "$color" ]; then
+        set_color $color $effect
+        echo -ne "${output}"
+        reset_color
+    else
+        echo -ne "$output"
+    fi
+}
+
+function eol {
+    sed 's/$/\x1B[K/'
+}
+
+function print_eol {
+    echo -e '\x1B[K'
+}
+
+
+
+function print {
+    local output="$1"
+    local color=${2:-""}
+    local char_limit=${3:-$fold}
+    local line_limit=${4:-0}
+    
+    local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit"
+    [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit"
+
+    eval "$cmd" | eol
+}
+
+
+function print_hline {
+    local color=${1:-"gray"}
+    local cols=${COLUMNS:-$(tput cols)}
+
+    decorate $(printf '%*s' $cols '' | tr ' ' -) "$color" 
+    echo | eol
+}
+
+function print_table {
+    # The problem here is that all escapes for colors are treated as normal visible symbols
+    sed "s/ ::: /@/g" | column -t -s "@" -o "    " | eol
+}
+
+function decorate_table {
+    print_hline
+    cat -
+    print_hline
+}
+
+function print_status {
+    local status="$1"
+
+    if [[ $status =~ ^.$ ]]; then
+        case $status in
+          0)
+            decorate "*" "red"
+            ;;
+          1)
+            decorate "*" "green"
+            ;;
+          2)
+            decorate "*" "yellow"
+            ;;
+          *)
+            decorate "$status" "red"
+            ;;
+        esac    
+    else
+        decorate "?" "red"
+    fi
+    echo -n " "
+}
+
+clear
diff --git a/lib/remote/report.sh b/lib/remote/report.sh
new file mode 100644
index 0000000..e6766a4
--- /dev/null
+++ b/lib/remote/report.sh
@@ -0,0 +1,51 @@
+function simple_report {
+    local generator="$1"
+    local filter="${2:-cat}"
+    {
+        eval "$generator 12>/dev/fd/12 | $filter"
+        report=$(cat<&12)
+    } 12<<EOF
+EOF
+    wait
+}
+
+
+function short_report {
+    local generator="$1"
+    local filter="${2:-print_table | decorate_table}"
+
+    configure_palete "$palete"
+    simple_report "$generator" "$filter"
+    finish
+}
+
+
+function standart_report {
+    local generator="$1"
+    local filter="${2:-print_table | decorate_table}"
+
+    configure_palete "$palete"
+    simple_report "$generator" "$filter"
+
+    print "$report"
+    echo
+    finish
+}
+
+
+function report {
+        # Eval breaks output for some reason
+    case $report in
+      simple_report)
+        simple_report "$@"
+        ;;
+      short_report)
+        short_report "$@"
+        ;;
+      standart_report)
+        standart_report "$@"
+        ;;
+      default)
+        echo "Report is not implemented"
+    esac
+}
diff --git a/lib/remote/run.sh b/lib/remote/run.sh
new file mode 100644
index 0000000..fdd7311
--- /dev/null
+++ b/lib/remote/run.sh
@@ -0,0 +1,13 @@
+function run_ {
+    local output=$(eval source "$@")
+
+    flock -x $0 echo -e "$output"
+}
+
+function run {
+    if [ $parallel -gt 0 ]; then
+        run_ "$@" &
+    else
+        run_ "$@"
+    fi
+}
diff --git a/lib/remote/status.sh b/lib/remote/status.sh
new file mode 100644
index 0000000..7bef6c8
--- /dev/null
+++ b/lib/remote/status.sh
@@ -0,0 +1,196 @@
+. setup/remote.sh
+. lib/remote/print.sh
+. lib/remote/report.sh
+. lib/remote/run.sh
+    
+function check_server_traffic {
+    local result="$1"
+    local yellow=${2:-0}
+    local red=${3:-0}
+    
+    local name=("in" "out" "fwd")
+
+    local traf    
+    read -ra traf <<< $result
+    if [ ${#traf[@]} -ne 3 ]; then
+        [ -z "$result" ] && result="No info received"
+        print "$result" "red" $fold 1 
+        return
+    fi
+    
+    local output=""
+    for i in $(seq 0 2); do
+        if ! [[ ${traf[$i]} =~ ^[0-9]+$ ]]; then
+            print "$result" "red" $fold 1
+            return
+        fi
+
+        local val=$(printf "% 4u" $((${traf[$i]} / 1024 / 1024 / 1024)))
+        if [ $red -gt 0 -a $val -gt $red ]; then
+            val=$(decorate "$val GB" "red")
+        elif [ $yellow -gt 0 -a $val -gt $yellow ]; then
+            val=$(decorate "$val GB" "yellow")
+        else
+            val="$val GB"
+        fi
+        
+#        output="$output${output:+, }${name[$i]}: $val"
+        output="$output${output:+, } $val"
+    done
+    
+#    print "$output"
+    print "in/out/fwd: $output"
+}
+
+function check_server_ {
+    local service=$1 && shift
+
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+
+    local result=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_$service.sh 2>&1 )
+
+    case "$service" in
+      'traffic')
+        check_server_traffic "$result" "$@"
+        ;;
+      *)
+        print "$result"
+    esac        
+
+}
+
+
+function check_server_status {
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+    local services="$@"
+
+    local ping_output=$(scripts/ping.pl $host $port $timeout)
+    print_status "$ping_output"
+
+    if [ "$services" == "-" ]; then return; fi
+
+    if [ "$ping_output" == "1" ]; then
+        local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1)
+
+        local status=1
+        [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n"
+        print_status $status
+
+        # Check Additional services, error is reported if anything provides information
+        local headers=""
+        if [ -n "$services" ]; then
+            for service in "$services"; do 
+                if [[ "$service" =~ ^vpn/(.*)$ ]]; then
+                    local ip=${BASH_REMATCH[1]}
+                    [ $(hostname) == "styx" ] && check_server_status "$ip" "-"
+                else
+                    local service_status=1
+                    local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d')
+                    if [ -n "$service_output" ]; then
+                        first_line=$(echo "$service_output" | head -n 1)
+                        if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then
+                            service_status=${BASH_REMATCH[1]}
+                            service_output=$(sed "1 d" <<< "$service_output")
+                            if [ -n "${BASH_REMATCH[2]}" ]; then
+                                service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}")       #"
+                                headers="$headers $service_header"
+                            fi
+                        else
+                            service_status=0
+                        fi
+
+                        if [ -n "$service_output" ]; then
+                            [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")"
+                            output="${output}\n  $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/  /')"
+                        fi
+                    fi
+                fi
+
+            done
+        fi
+
+        print " ::: ${headers}"
+        #report="$report<section>$output"
+        if [ -n "$output" ]; then
+            flock -x $0 echo "${output}" >&12
+        fi 
+    else
+        print_status "x"
+        for service in "$services"; do 
+            print_status "x"
+        done
+    fi
+}
+
+function check_service {
+    local service="$1" && shift
+    local id="$1" && shift
+    local args="$@"
+    
+    local output=$(service/check_${service}.sh ${id} "$@" 2>&1  | sed 's/^[[:space:]]*//;/^$/d')
+    
+    local info=$(echo "$output" | tail -n 1)
+    
+    local online=0
+    local health=0
+    local header=""
+    if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then
+        online=${BASH_REMATCH[1]}
+        health=${BASH_REMATCH[2]}
+        header=${BASH_REMATCH[3]}
+
+        output=$(sed '$ d' <<< "$output")
+    fi
+    
+    print_status "$online"
+    print_status "$health"
+
+    header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header")                # "
+    print " ::: ${header}"
+
+
+    important=$(grep "^\*" <<< "$output")
+    messages=$(grep -v "^\*" <<< "$output")
+
+    if [ -n "$output" ]; then
+        output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n"
+        flock -x $0 echo "${output}" >&12
+    fi
+}
+
+
+function check__ {
+    local args    
+    local title="$1" && shift
+    read -ra args <<< "$1" && shift
+    local host=${args[0]};
+
+    print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; "
+    
+    local service
+    for service in "$@"; do
+        service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service")                  # "
+        ( eval "$service" )
+    done
+    print_eol
+}
+
+function check_ {
+    # Buffer the output
+    local output=$(check__ "$@")
+    echo -e "$output"
+}
+
+function check {
+    if [ $parallel -gt 0 ]; then
+        check_ "$@" &
+    else
+        check_ "$@"
+    fi
+}