Provision monitoring scripts on request

11 files changed, 79 insertions, 1 deletions

diff --git a/docs/consistency.txt b/docs/consistency.txt
index 1b7a600..caaaf36 100644
--- a/docs/consistency.txt
+++ b/docs/consistency.txt
@@ -43,6 +43,7 @@ Networking
 
 ADEI
 ====
+ - MySQL replication is working
  - No caching pods are hung (for whatever reason)
 
   
\ No newline at end of file
diff --git a/opts.sh b/opts.sh
index 25d232a..d44398f 100644
--- a/opts.sh
+++ b/opts.sh
@@ -62,6 +62,8 @@ Actions:
 
  Host system managment
     software            - Install additionaly configured software
+    monitor             - Install monitoring scripts
+    backup              - Install backup scripts
     current             - Current managmenet playbook with various temorary actions
 
  Custom actions
diff --git a/playbooks/ands-backup.yml b/playbooks/ands-backup.yml
new file mode 100644
index 0000000..fbfec4c
--- /dev/null
+++ b/playbooks/ands-backup.yml
@@ -0,0 +1,9 @@
+- name: Common setup procedures
+  hosts: ands
+  roles:
+    - role: ands_facts
+
+- name: Configure Backup
+  hosts: ands_servers
+  roles:
+    - role: ands_backup
diff --git a/playbooks/ands-monitor.yml b/playbooks/ands-monitor.yml
new file mode 100644
index 0000000..5e06604
--- /dev/null
+++ b/playbooks/ands-monitor.yml
@@ -0,0 +1,4 @@
+- name: Configure Monitoring
+  hosts: ands_servers
+  roles:
+    - role: ands_monitor
diff --git a/roles/ands_monitor/defaults/main.yml b/roles/ands_monitor/defaults/main.yml
new file mode 100644
index 0000000..ee3cb7b
--- /dev/null
+++ b/roles/ands_monitor/defaults/main.yml
@@ -0,0 +1 @@
+ands_script_path: "/opt/scripts"
diff --git a/roles/ands_monitor/tasks/main.yml b/roles/ands_monitor/tasks/main.yml
new file mode 100644
index 0000000..ac70d28
--- /dev/null
+++ b/roles/ands_monitor/tasks/main.yml
@@ -0,0 +1,9 @@
+- name: Create scripts directory
+  file: path="{{ ands_script_path }}" state=directory
+
+- name: "Deploy scripts"
+  template: src="{{ item | quote }}" dest="{{ ands_script_path }}/{{ script_name }}" owner=root group=root mode=0755
+  vars:
+    script_name: "{{ item | basename | regex_replace('\\.j2','') }}"
+  with_fileglob:
+    - "{{ role_path }}/templates/scripts/*.j2"
diff --git a/roles/ands_monitor/templates/scripts/check_server_status.sh.j2 b/roles/ands_monitor/templates/scripts/check_server_status.sh.j2
new file mode 100755
index 0000000..caa63ce
--- /dev/null
+++ b/roles/ands_monitor/templates/scripts/check_server_status.sh.j2
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+fs=`df -lm / | grep -vi Filesystem | sed -e 's/[[:space:]]\+/ /g' | cut -d ' ' -f 4`
+datafs=`df -lm /mnt/ands | grep -vi Filesystem | sed -e 's/[[:space:]]\+/ /g' | cut -d ' ' -f 4`
+mem=`free -g | grep "Mem" | sed -e 's/[[:space:]]\+/ /g' | cut -d ' ' -f 7`
+cpu=`uptime | sed -e "s/[[:space:]]/\n/g" | tail -n 1`
+
+if [ $fs -le 8192 ]; then
+    echo "Only $(($fs / 1024)) GB left in the root file system"
+fi
+
+if [ $datafs -le 1048576 ]; then
+    echo "Only $(($datafs / 1024)) GB left in the data file system"
+fi
+
+if [ $mem -le 16 ]; then
+    echo "The system is starving on memory, $mem GB left free"
+fi
+
+if [ `echo "$cpu < 20" | bc` -eq 0 ]; then
+    echo "The system is starving on cpu, $cpu is load average for the last 15 min"
+fi
+
+vol=$(/opt/MegaRAID/storcli/storcli64 /c0/v0 show | grep -P "^0/0" | grep "Optl" | wc -l)
+if [ -z "$vol" -o "$vol" -eq 0 ]; then
+    echo "Raid volume is not optimal:"
+    /opt/MegaRAID/storcli/storcli64 /c0/v0 show | grep -P "^0/0"
+fi
+
+disks=$(/opt/MegaRAID/storcli/storcli64 /c0 show | grep -P "(HDD|SSD)" | grep -v "Onln" | wc -l)
+if [ -z "$disks" -o "$disks" -ne 0 ]; then
+    echo "Not all disks are online:"
+    /opt/MegaRAID/storcli/storcli64 /c0 show | grep -P "(HDD|SSD)" | grep -v "Onln"
+fi
diff --git a/roles/ands_monitor/templates/scripts/check_uptime_status.sh.j2 b/roles/ands_monitor/templates/scripts/check_uptime_status.sh.j2
new file mode 100755
index 0000000..0602fcb
--- /dev/null
+++ b/roles/ands_monitor/templates/scripts/check_uptime_status.sh.j2
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+up=$(uptime | cut -d ' ' -f 4- | cut -d ',' -f 1 | sed -re 's/^\s*//')
+load=$(uptime | cut -d ' ' -f 4- | cut -d ',' -f 4- | cut -d ':' -f 2 | cut -d ',' -f 3 | sed -re 's/^\s*//')
+disks=$(/opt/MegaRAID/storcli/storcli64 /c0 show | grep -P "(HDD|SSD)" | grep "Onln" | wc -l)
+data=`df -lh /mnt/ands | grep -vi Filesystem | sed -e 's/[[:space:]]\+/ /g' | cut -d ' ' -f 4`
+
+echo -n "1 Up $up \${color gray}/ $disks disks, $data free, load: $load"
diff --git a/scripts/gluster.sh b/scripts/gluster.sh
index 07ca7f9..a3ff186 100755
--- a/scripts/gluster.sh
+++ b/scripts/gluster.sh
@@ -11,7 +11,7 @@ shift
 function info {
     vol=$1
 
-    status=$(gluster volume info databases | grep -P 'Status' | awk '{ print $2 }' | tr -d '\r\n')
+    status=$(gluster volume info "$vol" | grep -P 'Status' | awk '{ print $2 }' | tr -d '\r\n')
     bricks=$(gluster volume info "$vol"  | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n')
     avail=$(gluster volume status "$vol" detail | grep Brick | wc -l)
     online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l)
diff --git a/setup.sh b/setup.sh
index 8d67d6d..090bd7d 100755
--- a/setup.sh
+++ b/setup.sh
@@ -155,6 +155,12 @@ case "$action" in
     current)
         apply playbooks/current.yml "$@" || exit
         ;;
+    monitor)
+        apply playbooks/ands-monitor.yml "$@" || exit 1
+        ;;
+    backup)
+        apply playbooks/ands-backup.yml "$@" || exit 1
+        ;;
     setup)
         subrole=$2
         shift
diff --git a/setup/projects/adei/vars/globals.yml b/setup/projects/adei/vars/globals.yml
index 3da00c6..8638e44 100644
--- a/setup/projects/adei/vars/globals.yml
+++ b/setup/projects/adei/vars/globals.yml
@@ -9,6 +9,10 @@ adei_pod_env:
       value: "/data/home"
     - name: "SHELL"
       value: "/bin/bash"
+    - name: "MYSQL_MASTER_SERVER"
+      value: "mysql-master.adei.svc.cluster.local"
+    - name: "MYSQL_SLAVE_SERVER"
+      value: "mysql-slave.adei.svc.cluster.local"
     - name: "MYSQL_PORT"
       value: "3306"
     - name: "MYSQL_USER"