Script to fullhy clean-up outdated rc

3 files changed, 258 insertions, 0 deletions

diff --git a/scripts/maintain/gluster/gluster.sh b/scripts/maintain/gluster/gluster.sh
new file mode 100755
index 0000000..a3ff186
--- /dev/null
+++ b/scripts/maintain/gluster/gluster.sh
@@ -0,0 +1,177 @@
+#! /bin/bash
+
+. opts.sh
+
+[ $? -ne 0 -o -z "$gpod" ] && { echo "No storage pods are running..." ; exit 1 ; }
+[ -z "$1" ] && { echo "Usage: $0 <info|heal|migrate> <vol|all> [src] [dst]" ; exit 1 ; }
+action=$1
+shift
+
+
+function info {
+    vol=$1
+
+    status=$(gluster volume info "$vol" | grep -P 'Status' | awk '{ print $2 }' | tr -d '\r\n')
+    bricks=$(gluster volume info "$vol"  | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n')
+    avail=$(gluster volume status "$vol" detail | grep Brick | wc -l)
+    online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l)
+    
+    echo "Volume $vol: $status (Bricks: $bricks, Available: $avail, Online: $online)"
+}
+
+function heal {
+    vol=$1
+
+    distributed=0
+    gluster volume info "$vol" | grep "Type:" | grep -i "Distribute" &> /dev/null
+    [ $? -eq 0 ] && distributed=1
+
+    echo "Healing volume $vol"
+    echo "-------------------"
+    gluster volume heal "$vol" full
+    gluster volume heal "$vol" info
+    
+    if [ $distributed -eq 1 ]; then
+        echo "Rebalancing distributed volume $vol"
+        gluster volume rebalance "$vol" fix-layout start
+    fi
+    
+    
+    gluster volume status "$vol"
+}
+
+function migrate {
+    vol=$1
+    src=$2
+    dst=$3
+    
+    [ -z "$src" -o -z "$dst" ] && { echo "Source and destination servers are required" ; exit 1 ; }
+
+    src_brick=$(gluster volume info $vol | grep -P '^Brick\d+:' | awk '{ print $2 }' | grep -P "^$src" | tr -d '\r\n' )
+    dst_brick=${src_brick/$src/$dst}
+
+    [ -z "$src_brick" -o -z "$dst_brick" ] && return 0
+
+    echo "Volume $vol: migrating failed brick"
+    echo "             from $src_brick"
+    echo "               to $dst_brick"
+    echo "Press enter to continue"
+    read
+    [ $? -ne 0 ] && exit
+
+    gluster volume replace-brick $vol "$src_brick" "$dst_brick"  commit force
+    heal $vol
+}
+    
+
+function transport {
+    vol=$1
+    transport=${2:-tcp,rdma}
+    echo "Changing $vol to transport $transport"
+    gluster volume stop "$vol"
+    gluster volume set "$vol" config.transport "$transport"
+    gluster volume start "$vol"
+}
+
+
+
+function restart {
+    vol=$1
+
+    echo $vol
+    bricks=$(gluster volume info "$vol"  | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n')
+    online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l)
+
+    if [ "$bricks" -ne "$online" ]; then
+        echo "Restarting $vol ($online bricks of $bricks are/is online)"
+        gluster --mode=script volume stop "$vol" 
+        gluster --mode=script volume start "$vol"
+    fi
+}    
+
+function delete_failed {
+    vol=$1
+
+    bricks=$(gluster volume info "$vol"  | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n')
+    online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l)
+
+    if [ "$online" == "0" ]; then
+        echo "Deleting $vol ($online bricks of $bricks are/is online)"
+#        gluster --mode=script volume stop "$vol" 
+        gluster --mode=script volume delete "$vol" 
+    fi
+}
+
+
+function lvm_clean {
+    used_bricks=`gluster volume info | grep "/brick_" | sed -r  -e 's/.*brick_(.*)\/brick/\1/'`
+    
+    for ip in $(seq 1 3); do
+        echo "Node $ip"
+        echo "========"
+        lvm_bricks=`node $ip lvscan | grep brick  | sed -r -e 's/.*brick_([0-9a-z]*)[^0-9a-z].*/\1/'`
+    
+        diff=$(echo $used_bricks $lvm_bricks | tr -d '\r' | tr ' ' '\n' | sort | uniq -u)
+        remove=$(echo "$diff $lvm_bricks" | tr -d '\r' | tr ' ' '\n' | sort | uniq -d)
+
+        for id in $remove; do
+            echo "Removing ---------------------------------------------"
+            node $ip lvs -o name,time,size -S "'name =~ $id'"
+            echo "Removing ---------------------------------------------"
+            node $ip lvremove -y -S "'name =~ $id'"
+        done
+    done    
+    
+    
+}
+
+function lvm_remove_today {
+    for ip in $(seq 1 3); do
+        node $ip hostname
+#lvdisplay -o name,time -S 'time since "2018-03-16"'
+    done
+}
+
+
+function heketi_cmd {
+    heketi "$@"
+}
+
+function heketi_clean {
+    heketi_vols=`heketi topology info | grep "Name: vol_" | sed -r  -e 's/.*(vol_[0-9a-z]+)\s*$/\1/'`
+    gluster_vols=`gluster volume info | grep "Name: vol_" | sed -r  -e 's/.*(vol_[0-9a-z]+)\s*$/\1/'`
+    echo $heketi_vols
+    
+    diff=$(echo $gluster_vols $heketi_vols | tr -d '\r' | tr ' ' '\n' | sort | uniq -u)
+    remove=$(echo "$diff $gluster_vols" | tr -d '\r' | tr ' ' '\n' | sort | uniq -d)
+
+    for vol in $remove; do
+        echo "Stopping and deleting volume $vol"
+        echo "---------------------------------"
+        gluster --mode=script volume stop "$vol" 
+        gluster --mode=script volume delete "$vol"
+    done
+}
+
+if [[ "$action" =~ ^heketi ]]; then
+    eval "$action" "$@"
+elif [[ "$action" =~ ^lvm ]]; then
+    eval "$action" "$@"
+elif [[ -z "$1" || "$1" =~ ^all ]]; then
+    all=0
+    [ "$1" == "all_heketi" ] && all=1
+    [[ "$1" =~ ^all ]] && shift
+
+    vols=$(gluster volume info | grep -P '^Volume Name' | awk '{ print $NF }' | tr '\r\n' '  ')
+    for vol in $vols; do
+        if [ $all -eq 0 ]; then
+            [[ "$vol" =~ [0-9] ]] && continue
+            [[ "$vol" =~ ^vol_ ]] && continue
+            [[ "$vol" =~ ^heketi ]] && continue
+        fi
+
+        eval "$action" "$vol" "$@"
+    done
+else
+    eval "$action" "$@"
+fi
diff --git a/scripts/maintain/gluster/opts.sh b/scripts/maintain/gluster/opts.sh
new file mode 100644
index 0000000..2f76d8e
--- /dev/null
+++ b/scripts/maintain/gluster/opts.sh
@@ -0,0 +1,21 @@
+function get_gluster_pod {
+    oc -n glusterfs get pods -l 'glusterfs=storage-pod' | grep Running | awk '{ print $1 }' | head -n 1
+}
+
+gpod=$(get_gluster_pod)
+
+function gluster {
+    oc -n glusterfs rsh po/$gpod gluster "$@"
+}
+
+
+function node {
+    ip=$1
+    shift
+    
+    ssh -xq root@192.168.26.$ip "$@"
+}
+
+function heketi {
+    node 1 heketi-cli -s  http://heketi-storage.glusterfs.svc.cluster.local:8080 --user admin --secret "$(oc get secret heketi-storage-admin-secret -n glusterfs -o jsonpath='{.data.key}' | base64 -d)" "$@"
+}
diff --git a/scripts/maintain/prunerc.sh b/scripts/maintain/prunerc.sh
new file mode 100644
index 0000000..166ede6
--- /dev/null
+++ b/scripts/maintain/prunerc.sh
@@ -0,0 +1,60 @@
+#! /bin/bash
+
+setup=""
+commit=0
+prune_stopped=0
+
+while [ -n "$1" ]; do
+  case "$1" in
+    -c | --commit ) commit=1; shift ;;
+    -s | --stopped ) prune_stopped=1; shift ;;
+    -n | --setup ) setup="-n $2"; shift 2;;
+    * ) echo "$0 [-n project] [--commit] [--stopped]"; break ;;
+  esac
+done
+
+
+dcs="$(oc $setup get dc | grep -v NAME)"
+
+while read -r dc; do
+    name=$(echo "$dc" | awk '{ print $1 }')
+    revision=$(echo "$dc" | awk '{ print $2 }')
+    dc_desired=$(echo "$dc" | awk '{ print $3 }')
+    dc_current=$(echo "$dc" | awk '{ print $4 }')
+    
+    rcname="$name-$revision"
+    rc="$(oc $setup get rc $rcname | grep -v NAME)"
+    desired=$(echo "$rc" | awk '{ print $2 }')
+    current=$(echo "$rc" | awk '{ print $3 }')
+    ready=$(echo "$rc" | awk '{ print $4 }')
+
+    if [ $dc_desired -ne $dc_current ]; then
+        [ $commit -eq 1 ] && echo "Skipping faulty dc/$name"
+        continue
+    elif [ $desired -ne $current -o $desired -ne $ready ]; then
+        [ $commit -eq 1 ] && echo "Skipping not completely ready dc/$name"
+        continue
+    elif [ $desired -eq 0 -a $prune_stopped -ne 1 ]; then
+        [ $commit -eq 1 ] && echo "Skipping stopped dc/$name (last one could be faulty)"
+        continue
+    fi
+
+
+    rcs="$(oc $setup get rc -l openshift.io/deployment-config.name=$name | grep -v NAME)"
+    
+    while read -r oldrc; do
+        oldname=$(echo "$oldrc" | awk '{ print $1 }')
+        desired=$(echo "$oldrc" | awk '{ print $2 }')
+
+        [ $oldname = $rcname ] && continue
+        [ $desired -ne 0 ] && continue
+        
+        if [ $commit -eq 1 ]; then
+            echo "Removing: $oldname (keeping $rcname[$ready])" 
+            oc $setup delete rc "$oldname"
+        else 
+            echo "Will remove: $oldname (keeping $rcname[$ready])" 
+        fi
+    done <<< "$rcs"
+    
+done <<< "$dcs"