Ashish Jaiswal Blog

A blog dedicated to community

DRBD Automatic Failover Without Cluster

I have written a script to do a failover of DRBD resources. This script will get the drbd start mount the partition, and start the nfsserver and samba server and the get virtual IP setup.

This script won’t be useful for many, which are having a mix envrionment. such as node having some resource as primary and some as secondary. Reason behind this, that we where not able to use pacemaker in our current enviornment. Though we are not using this in production enviornment.

Put this in your systemd bootup process. check this

Please use it and let me know.

drbd_auto.sh 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

#!/bin/bash
# Ashish Jaiswal
# Date 16th Dec 2014
# This script expects the primary host to be primary node
# for all the drbd resources.

# List of resource on the node.
res=$(cat /etc/drbd.d/*.res | grep resource | cut -d " " -f 2)

# eth1 is the interface which we going to assign floating IP
device_id=eth1

# This is the floating IP
ip="192.168.1.100"

function services () {
  # Declaring an array of the services, we want to be stop or start
  service_name=(nfsserver smb nmb)

  for var in ${service_name[@]}
  do
    systemctl is-active $var | grep ^active > /dev/null

    if [ $? -ne 0 ]; then
      systemctl $1 $var
    fi
  done
}

function check_ip {
  # Check whether the ip is present or not
  primary_ip=$(ip -o -4 addr show $device_id | awk -F '[ /]+' '/global/ {print $4}')

  # Add ip if is not present 
  if [ -z $primary_ip ]; then
   ip address add $ip dev $device_id
  fi
}

function mount_status {
  mountpoint -q /$1
  if [ $? -ne 0 ]; then
      mount /$1
  fi
}

function drbd_failover {
  drbdadm primary --force $1
  if [ $? -ne 0 ]; then
    echo -e "There seems to be some major problem,\nPlease login to `hostname` and fix the error." | mail -s "DRBD failover Unsuccessful"  ashish1099@gmail.com
    exit 1
  fi
}

for resource in $res; do
  # To get the current state of the resources.
  pri_roles=$(drbdadm role $resource  | cut -d "/" -f 1 | egrep "Unknown|Primary|Secondary")
  sec_roles=$(drbdadm role $resource  | cut -d "/" -f 2 | egrep "Unknown|Primary|Secondary")

  # If the node has a primary resources
  if [ $pri_roles = Primary ]
  then
    mount_status $resource
    services start
    check_ip
    echo "This is primary node for $resource resource on `hostname`"
  fi

  # If the node has a secondary resources
  if [ $pri_roles = Secondary ] || [ $pri_roles = Unknown ]
  then
    ping -c2 $ip > /dev/null && mountpoint -q /$resource
    if [ $? -ne 0 ]
    then
      if [ $sec_roles = Primary ]
      then
        echo "$resource is available on Primary, This is a secondary node."
      fi

      if [ $sec_roles = Unknown ]
      then
        drbd_failover $resource
        mount_status $resource
        services start
        check_ip
        logger "Successfully moved drbd resources $resource and started nfs and samba on `hostname`"
        echo "Successfully moved drbd resources $resource and started nfs and samba on `hostname`" | mail -s "DRBD Successful Failover"  ashish1099@gmail.com
      fi

      if [ $sec_roles = Secondary ]
      then
        echo -e "Split brain detected, Please fix manually\n\nAll these command need to be ran\n# drbdadm secondary <resource>\n# drbdadm connect --discard-my-data <resource>\n\nOn the other node (the split brain survivor), if its connection state is also StandAlone\n# drbdadm connect <resource>\n\nHere is drbd doc link http://www.drbd.org/users-guide/s-resolve-split-brain.html" | mail -s "DRBD SplitBrain Detected"  ashish1099@gmail.com
      fi
    else
      services stop
      logger "This is seconday node of $resource. So nothing to be done on this server, because Primary is Active at this moment"
    fi
  fi
done

Comments