Add remote services

6 files changed, 1058 insertions, 0 deletions

diff --git a/remote/lib/README.txt b/remote/lib/README.txt
new file mode 100644
index 0000000..f023131
--- /dev/null
+++ b/remote/lib/README.txt
@@ -0,0 +1,21 @@
+conky format
+============
+ - Server status
+    *                   ping works
+    *                   no output from 'check_server_status.sh' and additional listed checks 
+    [*]                 Checks for additional listed services 'check_<service>_status.sh'. 
+                        - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string).
+                        - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow.
+    [headers]             
+                        
+    ...                 Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format
+                        server:port[:service] text....
+    
+ - Extra info for DarkSoft servers
+   *                    VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here.
+        # #             Traffic on VPN interface from/to
+
+ - Server traffic
+    in/out/forw.        Yellow/Red if above specified threshold. No additional errors are reported here.
+    
+    
\ No newline at end of file
diff --git a/remote/lib/parameters.sh b/remote/lib/parameters.sh
new file mode 100644
index 0000000..0714a68
--- /dev/null
+++ b/remote/lib/parameters.sh
@@ -0,0 +1,8 @@
+# Configurable
+timeout=2
+fold=120
+#palete='blue'
+
+
+# Helpers
+ssh_params="-x -q -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -o ConnectTimeout=$timeout"
diff --git a/remote/lib/print.sh b/remote/lib/print.sh
new file mode 100644
index 0000000..465fce5
--- /dev/null
+++ b/remote/lib/print.sh
@@ -0,0 +1,168 @@
+# underlines    4;30,...        blinking 5:30,...
+# backgrounds:  40,...
+
+declare -A colors=(
+    [black]='0;30'
+    [red]='0;31'
+    [green]='0;32'
+    [orange]='0;33'
+    [blue]='0;34'
+    [purple]='0;35'
+    [cyan]='0;36'
+    [lightgray]='0;37'
+    [gray]='1;30'
+    [lightred]='1;31'
+    [lightgreen]='1;32'
+    [yellow]='1;33'
+    [lightblue]='1;34'
+    [lightpurple]='1;35'
+    [lightcyan]='1;36'
+    [white]='1;37'
+)
+
+background=""
+foreground=""
+
+
+
+
+# Fixing color map
+#echo -ne "\e[44m"
+#echo -ne '\x1B[K'
+
+function set_background {
+    if [ -n "$background" ]; then
+        echo -ne "$background"
+        echo -ne '\x1B[K'
+    fi
+}
+
+function set_color {
+    local color=${1:-""}
+    local effect=${2:-""}
+    
+    c=${colors[$color]} 
+
+    
+    case "$effect" in
+      'u')
+        c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/")
+        ;;
+    esac
+    
+    [ -n "$color" ] && echo -ne "\033[${c}m"
+    set_background
+}
+
+function reset_color {
+    echo -ne "\033[0m"
+
+    set_background
+    [ -n "$foreground" ] && set_color "$foreground"
+}
+
+function finish {
+    echo -ne "\033[0m"
+    echo
+}
+
+function configure_palete {
+    case $1 in
+      blue)
+        background="\e[1;44m"
+        foreground="white"
+        ;;
+      *)
+        # Yellow is invisible on white
+        colors[yellow]=${colors[orange]}
+        colors[cyan]=${colors[blue]}
+        ;;
+    esac
+    
+    clear
+    set_background
+    reset_color
+}
+
+function decorate {
+    local output="$1"
+    local color=${2:-""}
+    local effect=${3:-""}
+    
+    if [ -n "$color" ]; then
+        set_color $color $effect
+        echo -ne "${output}"
+        reset_color
+    else
+        echo -ne "$output"
+    fi
+}
+
+function eol {
+    sed 's/$/\x1B[K/'
+}
+
+function print_eol {
+    echo -e '\x1B[K'
+}
+
+
+
+function print {
+    local output="$1"
+    local color=${2:-""}
+    local char_limit=${3:-$fold}
+    local line_limit=${4:-0}
+    
+    local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit"
+    [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit"
+
+    eval "$cmd" | eol
+}
+
+
+function print_hline {
+    local color=${1:-"gray"}
+    local cols=${COLUMNS:-$(tput cols)}
+
+    decorate $(printf '%*s' $cols '' | tr ' ' -) "$color" 
+    echo | eol
+}
+
+function print_table {
+    # The problem here is that all escapes for colors are treated as normal visible symbols
+    sed "s/ ::: /@/g" | column -t -s "@" -o "    " | eol
+}
+
+function decorate_table {
+    print_hline
+    cat -
+    print_hline
+}
+
+function print_status {
+    local status="$1"
+
+    if [[ $status =~ ^.$ ]]; then
+        case $status in
+          0)
+            decorate "*" "red"
+            ;;
+          1)
+            decorate "*" "green"
+            ;;
+          2)
+            decorate "*" "yellow"
+            ;;
+          *)
+            decorate "$status" "red"
+            ;;
+        esac    
+    else
+        decorate "?" "red"
+    fi
+    echo -n " "
+}
+
+
+configure_palete "$palete"
diff --git a/remote/lib/report.sh b/remote/lib/report.sh
new file mode 100644
index 0000000..a5c16d4
--- /dev/null
+++ b/remote/lib/report.sh
@@ -0,0 +1,15 @@
+function standart_report {
+    local generator="$1"
+    local filter="${2:-print_table | decorate_table}"
+    {
+    
+        #check_darksoft 12>/dev/fd/12 | print_table #| column -t -o " "
+        eval "$generator 12>/dev/fd/12 | $filter"
+        report=$(cat<&12)
+    } 12<<EOF
+EOF
+
+    print "$report"
+    echo
+    finish
+}
diff --git a/remote/lib/status.sh b/remote/lib/status.sh
new file mode 100644
index 0000000..3c7fbec
--- /dev/null
+++ b/remote/lib/status.sh
@@ -0,0 +1,177 @@
+. lib/parameters.sh
+. lib/print.sh
+. lib/report.sh
+    
+function check_server_traffic {
+    local result="$1"
+    local yellow=${2:-0}
+    local red=${3:-0}
+    
+    local name=("in" "out" "fwd")
+
+    local traf    
+    read -ra traf <<< $result
+    if [ ${#traf[@]} -ne 3 ]; then
+        [ -z "$result" ] && result="No info received"
+        print "$result" "red" $fold 1 
+        return
+    fi
+    
+    local output=""
+    for i in $(seq 0 2); do
+        if ! [[ ${traf[$i]} =~ ^[0-9]+$ ]]; then
+            print "$result" "red" $fold 1
+            return
+        fi
+
+        local val=$(printf "% 4u" $((${traf[$i]} / 1024 / 1024 / 1024)))
+        if [ $red -gt 0 -a $val -gt $red ]; then
+            val=$(decorate "$val GB" "red")
+        elif [ $yellow -gt 0 -a $val -gt $yellow ]; then
+            val=$(decorate "$val GB" "yellow")
+        else
+            val="$val GB"
+        fi
+        
+#        output="$output${output:+, }${name[$i]}: $val"
+        output="$output${output:+, } $val"
+    done
+    
+#    print "$output"
+    print "in/out/fwd: $output"
+}
+
+function check_server_ {
+    local service=$1 && shift
+
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+
+    local result=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_$service.sh 2>&1 )
+
+    case "$service" in
+      'traffic')
+        check_server_traffic "$result" "$@"
+        ;;
+      *)
+        print "$result"
+    esac        
+
+}
+
+
+function check_server_status {
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+    local services="$@"
+
+    local ping_output=$(scripts/ping.pl $host $port $timeout)
+    print_status "$ping_output"
+
+    if [ "$services" == "-" ]; then return; fi
+
+    if [ "$ping_output" == "1" ]; then
+        local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1)
+
+        local status=1
+        [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n"
+        print_status $status
+
+        # Check Additional services, error is reported if anything provides information
+        local headers=""
+        if [ -n "$services" ]; then
+            for service in "$services"; do 
+                if [[ "$service" =~ ^vpn/(.*)$ ]]; then
+                    local ip=${BASH_REMATCH[1]}
+                    [ $(hostname) == "styx" ] && check_server_status "$ip" "-"
+                else
+                    local service_status=1
+                    local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d')
+                    if [ -n "$service_output" ]; then
+                        first_line=$(echo "$service_output" | head -n 1)
+                        if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then
+                            service_status=${BASH_REMATCH[1]}
+                            service_output=$(sed "1 d" <<< "$service_output")
+                            if [ -n "${BASH_REMATCH[2]}" ]; then
+                                service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}")       #"
+                                headers="$headers $service_header"
+                            fi
+                        else
+                            service_status=0
+                        fi
+
+                        if [ -n "$service_output" ]; then
+                            [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")"
+                            output="${output}\n  $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/  /')"
+                        fi
+                    fi
+                fi
+
+            done
+        fi
+
+        print " ::: ${headers}"
+        #report="$report<section>$output"
+        echo "${output}" >&12
+    else
+        print_status "x"
+        for service in "$services"; do 
+            print_status "x"
+        done
+    fi
+}
+
+function check_service {
+    local service="$1" && shift
+    local id="$1" && shift
+    local args="$@"
+    
+    local output=$(service/check_${service}.sh ${id} "$@" 2>&1  | sed 's/^[[:space:]]*//;/^$/d')
+    
+    local info=$(echo "$output" | tail -n 1)
+    
+    local online=0
+    local health=0
+    local header=""
+    if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then
+        online=${BASH_REMATCH[1]}
+        health=${BASH_REMATCH[2]}
+        header=${BASH_REMATCH[3]}
+
+        output=$(sed '$ d' <<< $ouput)
+    fi
+    
+    print_status "$online"
+    print_status "$health"
+
+    headers=$(sed -r "s/\\$\{color\s+([^}]*)\}/$(set_color \\1)/g" <<< "$header")                # "
+    print " ::: ${headers}"
+
+
+    important=$(grep "^\*" <<< $output)
+    messages=$(grep -v "^\*" <<< $output)
+    [ -n "$output" ] && output="\n$(decorate "$service:$id" "cyan" "u")\n$(set_color white)$important\n$(reset_color)$messages\n"
+}
+
+
+
+
+function check {
+    local args    
+    local title="$1" && shift
+    read -ra args <<< "$1" && shift
+    local host=${args[0]};
+
+    print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/$(set_color \\1)/g; "
+    
+    local service
+    for service in "$@"; do
+        service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service")                  # "
+        ( eval "$service" )
+    done
+    print_eol
+}
diff --git a/remote/lib/support.lua b/remote/lib/support.lua
new file mode 100644
index 0000000..984d677
--- /dev/null
+++ b/remote/lib/support.lua
@@ -0,0 +1,669 @@
+require "luarocks.require"
+require "md5"
+require "lfs"
+
+home = os.getenv( "HOME" )
+
+os.execute("mkdir -p " .. home .. "/.conky/requests");
+os.execute("mkdir -p " .. home .. "/.conky/info");
+c_timer_init = -1
+
+do
+ function conky_eval(...)
+    return conky_parse(table.concat(arg, " "))
+ end
+ 
+ function conky_multiply_line(times, ...)
+    local str = table.concat(arg, " ")
+    local tbl = {}
+    for i=1,times,1 do
+	tbl[i] = string.gsub(str,"@i@", i-1)
+    end
+    local res = table.concat(tbl,"\n")
+    return res;
+ end
+
+ function split(delimiter, text)
+  local list = {}
+  local pos = 1
+  if string.find("", delimiter, 1) then -- this would result in endless loops
+    error("delimiter matches empty string!")
+  end
+  while 1 do
+    local first, last = string.find(text, delimiter, pos)
+    if first then -- found?
+      table.insert(list, string.sub(text, pos, first-1))
+      pos = last+1
+    else
+      table.insert(list, string.sub(text, pos))
+      break
+    end
+  end
+  return list
+ end
+ 
+ function conky_filter_real_ip(iface)
+    local req =  string.format('${addrs %s}', iface)
+    local ips = conky_parse(req);
+--[[
+    print(ips)
+--]]
+    local list = split(",", ips)
+    for i, item in ipairs(list) do
+	if (string.match(item, "141\.52\.64")) then
+	    return item
+	end
+    end
+    return "Unknown"
+ end
+
+ user = split('/', home)
+ user = user[#user]
+
+ result_check_server_status = {}
+ result_check_adei_source = {}
+ result_check_server_ = {}
+ result_check_service_ = {}
+ online = {}
+ status = {}
+ tested = {}
+ c_timer = c_timer_init
+ width = 80
+ update_time = 0
+ outdate_time = 0
+ timeout = 2
+
+ function conky_set_width (w)
+    width = tonumber(w)
+    return ""
+ end
+ 
+ function conky_set_timeout (t)
+    timeout = tonumber(t)
+    return ""
+ end
+
+ function cmd_popen(server, port, cmd)
+    if (c_timer_init == 0) then
+	return io.popen(cmd)
+    else 
+	local cmd_md5 = md5.sumhexa(cmd)
+        local fn = string.format("%s/.conky/info/%s:%s-%s.srv", home, server, port, cmd_md5)
+	stat = lfs.attributes(fn)
+	if (stat == nil) then
+	    return nil
+	end
+	
+	local since = os.difftime(stat['modification'], update_time)
+	if (since < 0) then
+	    return nil
+	end
+	since = os.difftime(outdate_time, os.time())
+	if (since < 0) then
+	    return nil
+	end
+	return io.open(fn, "r")
+    end
+ end
+
+ function conky_check_server_status(server, port, ...)
+    port = port or 22
+    local name = string.format("%s:%i", server, port);
+
+    if (c_timer < 0) then
+	if (c_timer == -1) then
+	    local fn = string.format("%s/.conky/requests/%s:%i.srv", home, server, port)
+	    
+	    tested[string.format("%s:%i", server, port)] = false;
+
+	    local f = assert(io.open(fn, "w"))
+
+	    local cmd = string.format("/etc/conky/scripts/ping.pl %s:%i %i", server, port, timeout)
+	    f:write(cmd .. "\n")
+
+	    if (arg[1] ~= "-") then
+		cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_server_status.sh 2>&1 | fold -w %i", port, timeout, server, width)
+		f:write(cmd .. "\n")
+	    
+		for i, service in ipairs(arg)
+		do		
+		    cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_%s_status.sh 2>&1 | fold -w %i", port, timeout, server, service, width)
+	    	    f:write(cmd .. "\n")
+		end
+	    end
+	    f:close()
+	end    
+	if (result_check_server_status[name] == nil)  then
+	    local res
+	    if (math.abs(c_timer)%2 > 0) then
+		res = "${color yellow}?${color}"
+	    else
+		res = "${color yellow}.${color}"
+	    end
+	    
+	    if (arg[1] ~= "-") then
+		res = res .. "  "
+		for i, service in ipairs(arg) do
+	    	    res = res .. "  "
+		end
+	    end
+	    --result_check_server_status[server] = res
+	    --status[server] = false
+	    return res
+	end
+    elseif (c_timer == 0) then
+	result_check_server_status[name] = check_server_status(server, port, arg)
+    end
+
+    return result_check_server_status[name];
+ end
+ 
+ function check_server_status(server, port, additional)
+    port = port or 22
+
+    local res
+    local cmd = string.format("/etc/conky/scripts/ping.pl %s:%i %i", server, port, timeout)
+    local cmdf = cmd_popen(server, port, cmd)
+    if (cmdf == nil) then
+	res = -1
+    else
+	local data = cmdf:read("*a")
+	res = tonumber(data)
+	cmdf:close()
+    end
+    
+    if (additional[1] == "-") then
+	if (res > 0) then
+	    online[server] = true
+	    return "${color green}*"
+	elseif (res < 0) then
+	    return "${color red}?"
+	else
+	    online[server] = false
+	    return "${color red}*"
+	end
+    end
+    
+    if (res > 0) then
+	online[server] = true
+	
+	local output
+
+	cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_server_status.sh 2>&1 | fold -w %i", port, timeout, server, width)
+	cmdf = cmd_popen(server, port, cmd)
+	if (cmdf == nil) then
+	    output = string.format("${color green}* ${color red}?")
+	else
+	    local res = {}
+	    local line = cmdf:read("*l")
+	    while line do
+		local m1 = string.match(line, "^\(.*[^%s]\)%s*$")
+		if (m1) then
+	    	    table.insert(res, m1)
+		end
+		line = cmdf:read("*l")
+	    end
+--	    res = cmdf:read("*a")
+	    cmdf:close()
+	    
+	    res = table.concat(res, "\n")
+	    
+	    if (string.match(res, "[%a%d]")) then
+		table.insert(status, string.format("${color yellow}%s:%i${color gray}\n%s", server, port, res))
+	        output = string.format("${color green}* ${color red}*")
+	    else
+		output = string.format("${color green}* *")
+	    end
+	end
+
+	for i, service in ipairs(additional)
+	do	
+	    cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_%s_status.sh 2>&1 | fold -w %i", port, timeout, server, service, width)
+	    cmdf = cmd_popen(server, port, cmd)
+	    if (cmdf == nil) then
+		output = output .. string.format(" ${color red}?")
+	    else 
+	        local res = {}
+		local line = cmdf:read("*l")
+	        while line do
+		    local m1 = string.match(line, "^\(.*[^%s]\)%s*$")
+		    if (m1) then
+	    		table.insert(res, m1)
+		    end
+		    line = cmdf:read("*l")
+		end
+		cmdf:close()
+
+		if (res[1]) then
+		    local info = table.remove(res)
+		    local s = 0
+		    local extra = ""
+		    
+		    local m1, m2 = string.match(info, "^\(%d+\)\(%s.+\)$")
+		    if (m1 == nil) then
+			m1 = string.match(info, "^\%d+$");
+		    end
+		    if (m1 == nil) then
+			table.insert(res, info)
+		    else
+			s = tonumber(m1)
+			if (m2 ~= nil) then
+			    extra = string.sub(m2,2)
+			end
+		    end
+	    	    
+		    if (res[1]) then
+			res = table.concat(res, "\n")
+			table.insert(status, string.format("${color yellow}%s:%i:%s${color gray}\n%s", server, port, service, res))
+		    end
+		    if (s == 0) then
+			output = output .. string.format(" ${color red}*")
+		    elseif (s == 1) then
+			output = output .. string.format(" ${color green}*")
+		    else
+			output = output .. string.format(" ${color yellow}*")
+		    end
+		    output=output .. " ${color white}" .. extra;
+		else
+		    output = output .. string.format(" ${color green}*")
+		end
+	    end
+	end
+	
+	return output
+    else
+	if (res < 0) then
+	    res = string.format("${color red}?  ")
+	else
+	    online[server] = false
+	    res = string.format("${color red}*  ")
+	end
+	for i, service in ipairs(additional) do
+	    res = res .. "  "
+	end
+
+	return res
+    end
+ end
+
+ function conky_check_adei_source(server, port, config, setup, db_server, db_name, ignore_list)
+    port = port or 22
+    local name = string.format("%s:%i__%s__%s", server, port, db_server, db_name)
+    if (c_timer < 0) then
+	if (c_timer == -1) then
+	    ignore_list = ignore_list or ""
+
+	    local fn = string.format("%s/.conky/requests/%s:%i.srv", home, server, port)
+
+	    local f = assert(io.open(fn, "a+"))
+	    local cmd = string.format('ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_adei_source.sh %s %s %s %s "%s" 2>&1', port, timeout, server, config, setup, db_server, db_name, ignore_list)
+	    f:write(cmd .. "\n")
+	    f:close()
+	end
+	
+	if (result_check_adei_source[name] == nil) then
+	    return "${color yellow}?${color}"
+	else
+	    if (online[server]) then
+    		return result_check_adei_source[name];
+	    else
+		return "${color red}?${color}"
+	    end
+	end
+    elseif (online[server]) then
+	if (c_timer == 0) then
+	    result_check_adei_source[name] = check_adei_source(server, port, config, setup, db_server, db_name, ignore_list)
+	end
+        return result_check_adei_source[name];
+    else
+	return "${color red}?${color}"
+    end
+ end
+ 
+ function check_adei_source(server, port, config, setup, db_server, db_name, ignore_list)
+	ignore_list = ignore_list or ""
+	cmd = string.format('ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_adei_source.sh %s %s %s %s "%s" 2>&1', port, timeout, server, config, setup, db_server, db_name, ignore_list)
+	cmdf = cmd_popen(server, port, cmd)
+	if (cmdf == nil) then
+	    return "${color red}?"
+	end
+	
+	local res = {}
+	line = cmdf:read("*l")
+	while line do
+	    local m1 = string.match(line, "^\(.*[^%s]\)%s*$")
+	    if (m1) then
+		table.insert(res, m1)
+	    end
+	    line = cmdf:read("*l")
+	end
+	cmdf:close()
+
+	local info = table.remove(res)
+
+	local t1, t2, t3 = string.match(info, "^\(%d+\)%s+\(%d+\)%s+\(%d+\)")
+	
+	local s = tonumber(t1)
+	if (s == nil) then 
+	    table.insert(res, info)
+	    s = 0 
+	end
+
+	res = table.concat(res, "\n")
+	
+	local output
+	if (s == 1) then 
+	    output="${color green}*"
+	else
+--[[
+	    table.insert(status, string.format("${color yellow}%s -- %s${color gray}\n%s\n", db_server, db_name, res))
+--]]
+	    if (s == 0) then output="${color red}*"
+	    else output="${color yellow}*" end
+	end
+	
+--	if (s ~= 0) then
+	    local groups = tonumber(t2)
+	    if (groups) then
+		output = output .. string.format("${color white} %i groups", groups);
+	    else
+		output = output .. string.format("${color red} ? groups");
+	    end
+
+	    local size = tonumber(t3)	
+	    if (groups and size) then
+		output = output .. string.format("${color white}, %i GB", size);
+	    else
+		output = output .. string.format("${color red}, ? GB");
+	    end
+--	end
+
+        if (string.match(res, "[%a%d]")) then
+	    output = output .. "${color gray}\n   ${font Bitstream Vera Sans Mono:size=7}" .. string.gsub(res,"\n", "${font}\n   ${font Bitstream Vera Sans Mono:size=7}") .. "${font}${color white}"
+	end
+	
+	return output
+ end
+
+ function conky_check_server_(service, server, port, ...)
+    port = port or 22
+    local name = string.format("%s:%i:%s", server, port, service)
+    if (c_timer < 0) then
+        if (c_timer == -1) then
+            ignore_list = ignore_list or ""
+            
+            local fn = string.format("%s/.conky/requests/%s:%i.srv", home, server, port)
+        
+            local f = assert(io.open(fn, "a+"))
+            local cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_server_%s.sh 2>&1", port, timeout, server, service)
+            f:write(cmd .. "\n")
+            f:close()
+        end
+           
+        if (result_check_server_[name] == nil) then
+            return "${color yellow}?${color}"
+        else
+            if (online[server]) then
+                return result_check_server_[name];
+            else
+                return "${color red}?${color}"
+            end
+        end
+    elseif (online[server]) then
+        if (c_timer == 0) then
+            result_check_server_[name] = check_server_(service, server, port, arg)
+        end
+        return result_check_server_[name];
+    else
+        return "${color red}?${color}"
+    end
+ end
+
+ function conky_check_service_(service, id, ...)
+    local name = string.format("%s:%s", service, id)
+    if (c_timer < 0) then
+        if (c_timer == -1) then
+            ignore_list = ignore_list or ""
+
+	    tested[string.format("%s:%s", service, id)] = false;
+            
+            local fn = string.format("%s/.conky/requests/%s:%s.srv", home, service, id)
+        
+            local f = assert(io.open(fn, "a+"))
+            local cmd = string.format("/etc/conky/service/check_%s.sh %s %s 2>&1", service, id, table.concat(arg, " "))
+            f:write(cmd .. "\n")
+            f:close()
+        end
+           
+        if (result_check_server_[name] == nil) then
+            return "${color yellow}?${color}"
+        else
+            return result_check_server_[name];
+        end
+    else
+        if (c_timer == 0) then
+            result_check_server_[name] = check_service_(service, id, arg)
+        end
+        return result_check_server_[name];
+    end
+ end
+
+
+ function format_traffic(value, yellow, red)
+    yellow = yellow or 100
+    red = red or 1000
+    
+    local value = math.floor(tonumber(value) / 1073741824)
+    
+    local res
+    if (value > red) then res="${color red}"
+    elseif (value > yellow) then res="${color yellow}"
+    else res="${color white}" end
+
+    if (value < 10) then res = res .. "  " .. value
+    elseif (value < 100) then res = res .. " " .. value
+    else res = res .. value end
+    
+    return res .. " GB${color gray}"
+ end
+ 
+ function check_server_(service, server, port, opts)
+        port = port or 22
+	local cmd = string.format("ssh -x -p %i -o ConnectTimeout=%i root@%s /opt/scripts/check_server_%s.sh 2>&1", port, timeout, server, service)
+	cmdf = cmd_popen(server, port, cmd)
+	if (cmdf == nil) then
+	    return "${color red}?"
+	end
+
+	local res = {}
+	line = cmdf:read("*l")
+	while line do
+	    local m1 = string.match(line, "^\(.*[^%s]\)%s*$")
+	    if (m1) then
+		table.insert(res, m1)
+	    end
+	    line = cmdf:read("*l")
+	end
+	cmdf:close()
+	
+	if (service == "traffic") then
+	    if (res[1]) then
+		local t1, t2, t3 = string.match(res[1], "^\(%d+\)%s+\(%d+\)%s+\(%d+\)")
+		
+		if (t1 and t2 and t3) then
+	    	    if ((opts) and (opts[1])) then yellow = tonumber(opts[1])
+		    else yellow = nil end
+		    if ((opts) and (opts[2])) then red = tonumber(opts[2])
+		    else red = nil end
+		
+		    t1=format_traffic(t1,yellow,red)
+		    t2=format_traffic(t2,yellow,red)
+		    t3=format_traffic(t3,yellow,red)
+		    return string.format("${color gray}i: %s, o: %s, f: %s", t1, t2, t3)
+		else
+		    return "${color red}" .. res[1]
+		end
+	    end
+	end
+	
+
+	return ""
+ end
+
+
+ function check_service_(service, id, opts)
+	local cmd = string.format("/etc/conky/service/check_%s.sh %s %s 2>&1", service, id, table.concat(opts, " "))
+	cmdf = cmd_popen(service, id, cmd)
+	if (cmdf == nil) then
+	    return "${color red}?"
+	end
+
+	local res = {}
+	local local_res = {}
+	line = cmdf:read("*l")
+	while line do
+	    local t, m1 = string.match(line, "^\(\*?\)\(.*[^%s]\)%s*$")
+	    if (m1) then
+		if (t == nil) or (t == '') then
+		    table.insert(res, m1)
+		else
+		    table.insert(local_res, string.sub(m1,1))
+		end
+	    end
+	    line = cmdf:read("*l")
+	end
+	cmdf:close()
+
+	local output
+	local info = table.remove(res)
+	local t1 = 0
+	local t2 = nil
+	local msg = ''
+
+	if (info) then
+	    t1, t2, msg = string.match(info, "^\(%d+\)%s+\(%d+\)\(.*\)")
+	end
+	
+	local s = tonumber(t1)
+	if (s == nil) then 
+	    table.insert(res, info)
+	    s = 0 
+	end
+
+	if (s == 1) then 
+	    output="${color green}*"
+	else
+	    if (s == 0) then output="${color red}*"
+	    else output="${color yellow}*" end
+	end
+	
+	s = tonumber(t2)
+	if (s == nil) then 
+	    s = 0 
+	else
+	    if (s == 1) then 
+		output=output .. " ${color green}*"
+	    else
+		if (s == 0) then output=output .. " ${color red}*"
+		else output=output .. " ${color yellow}*" end
+	    end
+	end
+
+	output = output .. " ${color white}" .. msg
+
+	res = table.concat(res, "\n")
+	local_res = table.concat(local_res, "\n")
+	
+
+	if (string.match(res, "[%a%d]")) then
+	    table.insert(status, string.format("${color yellow}%s:%s${color gray}\n%s", service, id, res))
+	end
+
+	if (string.match(local_res, "[%a%d]")) then
+	    output = output .. "${color gray}\n   ${font Bitstream Vera Sans Mono:size=7}" .. string.gsub(local_res,"\n", "${font}\n   ${font Bitstream Vera Sans Mono:size=7}") .. "${font}${color white}"
+	end
+	
+	return output
+ end
+
+
+ 
+ function conky_print_server_errors(interval, tag)
+    local res = table.concat(status, "\n")
+
+    if (c_timer < 0) then
+	tag = user .. "_" .. tag
+
+	if (c_timer == -1) then
+	    update_time = os.time()
+	    
+	    local result = 0;
+	    while (result == 0) do
+	        result = os.execute("ps x | grep conky_process_requests | grep " .. tag .. " | grep -v grep > /dev/null")
+	    end
+
+	    for item, s in pairs(tested) do
+		os.execute("/etc/conky/scripts/conky_process_requests.pl \"" .. item .. ".srv\" " .. tag .. " &")
+	    end
+	end
+	
+	local result=os.execute("ps x | grep conky_process_requests | grep " .. tag .. " | grep -v grep > /dev/null")
+	if (result == 0) then
+	    c_timer = c_timer - 1;
+	    local since = os.difftime(os.time(), update_time)
+	    if (since > tonumber(interval)) then
+		result_check_server_status = {}
+		result_check_adei_source = {}
+		result_check_server_ = {}
+	    end
+	else
+	    outdate_time = os.time() + 2 * tonumber(interval)
+	    c_timer = 0
+	end
+	return ""
+    else 
+	c_timer = c_timer + conky_info.update_interval
+	if (c_timer > tonumber(interval)) then 
+	    status = {}
+	    tested = {}
+	    c_timer = c_timer_init
+	end
+    end
+
+    if (string.match(res, "[%a%d]")) then
+	return string.format("\n\n%s", res)
+    else
+	return ""
+    end
+ end
+ 
+ function conky_outcon(pos1, pos2)
+    local n = tonumber(conky_parse('${tcp_portmon 32768 65535 count}'))
+    local res = {}
+    local str = ""
+    for i=1,n do
+	local val = tostring(conky_parse(string.format('${tcp_portmon 32768 65535 rhost %i}:${tcp_portmon 32768 65535 rport %i}',i - 1,i - 1)))
+	
+	if (res[val] == nil) then
+	    res[val] = 1
+	else
+	    res[val] = res[val] + 1
+	end
+    end
+
+
+    local names = {}
+    for addr,num in pairs(res) do
+	table.insert(names, addr)
+    end
+
+    table.sort(names, function(a,b) return res[a]>res[b] end)
+
+    for i=1,#names do
+	local name = names[i]
+	str = str .. string.format("${goto %i}%s${goto %i}cnt  %3i\n",pos1,name,pos2,res[name])
+    end
+    
+    return str
+ end
+end