From e946fe8df5f413959ce4e9bc5577f7101ed5b795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8Cajka?= Date: Mon, 21 Jan 2019 10:43:55 +0100 Subject: [PATCH] Inital commit --- Dockerfile | 32 ++++ conf/hello-openshift-template.json | 101 ++++++++++ conf/settings.sh | 111 +++++++++++ lib/config-generators.sh | 297 +++++++++++++++++++++++++++++ lib/failover-functions.sh | 79 ++++++++ lib/utils.sh | 269 ++++++++++++++++++++++++++ makefile.test | 11 ++ monitor.sh | 15 ++ tests/echoserver.py | 36 ++++ tests/verify_failover_image.sh | 90 +++++++++ 10 files changed, 1041 insertions(+) create mode 100644 Dockerfile create mode 100644 conf/hello-openshift-template.json create mode 100755 conf/settings.sh create mode 100755 lib/config-generators.sh create mode 100755 lib/failover-functions.sh create mode 100755 lib/utils.sh create mode 100644 makefile.test create mode 100755 monitor.sh create mode 100644 tests/echoserver.py create mode 100755 tests/verify_failover_image.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3a001fa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# +# VIP failover monitoring container for OpenShift. +# +# ImageName: openshift/origin-keepalived-ipfailover +# +FROM registry.fedoraproject.org/f29/origin-base:latest + +ENV NAME=origin-keepalived-ipfailover \ + VERSION=3.11 \ + ARCH=x86_64 + +RUN INSTALL_PKGS="kmod keepalived iproute psmisc nmap-ncat net-tools ipset ipset-libs" && \ + dnf install -y $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf clean all +COPY . /var/lib/ipfailover/keepalived/ + +LABEL io.k8s.display-name="OpenShift IP Failover" \ + io.k8s.description="This is a component of OpenShift and runs a clustered keepalived instance across multiple hosts to allow highly available IP addresses." \ + io.openshift.tags="openshift,ha,ip,failover" \ + summary="This is the base image from which all OpenShift Origin images inherit." \ + maintainer="Jakub Cajka " \ + License="GPLv2+" \ + name="$FGC/$NAME" \ + com.redhat.component="$NAME" \ + version="$VERSION" \ + architecture="$ARCH" \ + usage="This is the base image from which all OpenShift Origin images inherit." + +EXPOSE 1985 +WORKDIR /var/lib/ipfailover +ENTRYPOINT ["/var/lib/ipfailover/keepalived/monitor.sh"] diff --git a/conf/hello-openshift-template.json b/conf/hello-openshift-template.json new file mode 100644 index 0000000..fcec98c --- /dev/null +++ b/conf/hello-openshift-template.json @@ -0,0 +1,101 @@ +{ + "kind": "Config", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "hello-openshift-service", + "creationTimestamp": null + }, + "spec": { + "ports": [ + { + "protocol": "TCP", + "port": 8080, + "targetPort": 0, + "nodePort": 0 + } + ], + "selector": { + "name": "hello-openshift-label" + }, + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + }, + { + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "hello-openshift", + "creationTimestamp": null, + "labels": { + "name": "hello-openshift-label" + } + }, + "spec": { + "replicas": 1, + "selector": { + "name": "hello-openshift-label" + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "name": "hello-openshift-label" + } + }, + "spec": { + "containers": [ + { + "name": "hello-openshift-container", + "image": "openshift/hello-openshift", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {}, + "securityContext": { + "capabilities": {}, + "privileged": false + } + } + ], + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst" + } + } + }, + "status": { + "replicas": 0 + } + }, + { + "kind": "Route", + "apiVersion": "v1", + "metadata": { + "name": "hello-openshift-route", + "creationTimestamp": null + }, + "spec": { + "host": "hello.openshift.test", + "to": { + "kind": "Service", + "name": "hello-openshift-service" + } + }, + "status": {} + } + ] +} diff --git a/conf/settings.sh b/conf/settings.sh new file mode 100755 index 0000000..f5d3b04 --- /dev/null +++ b/conf/settings.sh @@ -0,0 +1,111 @@ +#!/bin/bash + + +# ======================================================================== +# Settings passed by the failover coordinator on OpenShift Origin. +# ======================================================================== + +# Name of this IP Failover config instance. +HA_CONFIG_NAME="${OPENSHIFT_HA_CONFIG_NAME:-"OpenShift-IPFailover"}" + +# IP Failover config selector. +HA_SELECTOR="${OPENSHIFT_HA_SELECTOR:-""}" + + +# List of virtual IP addresses. +# +# The value entries are comma-separated entries of the form: +# +# +# where: ipaddress-range = - +# +# Example: +# OPENSHIFT_HA_VIRTUAL_IPS="10.42.42.42,10.100.1.20-24" +# +HA_VIPS="${OPENSHIFT_HA_VIRTUAL_IPS:-""}" +HA_VIP_GROUPS="${OPENSHIFT_HA_VIP_GROUPS:-""}" + +# Interface (ethernet) to use - bound by vrrp. +NETWORK_INTERFACE="${OPENSHIFT_HA_NETWORK_INTERFACE:-""}" # "enp0s8" + + +# Service port to monitor for failover. +HA_MONITOR_PORT="${OPENSHIFT_HA_MONITOR_PORT:-"80"}" + +# Number of initial replicas. +HA_REPLICA_COUNT="${OPENSHIFT_HA_REPLICA_COUNT:-"1"}" + + +# Offset value to use to set the virtual router ids. Using different offset +# values allows multiple ipfailover configurations to exist within the +# same cluster. Range 1..255 +# HA_VRRP_ID_OFFSET=30 +# +HA_VRRP_ID_OFFSET="${OPENSHIFT_HA_VRRP_ID_OFFSET:-"0"}" + +# When the DC supplies an (non null) iptables chain +# (OPENSHIFT_HA_IPTABLES_CHAIN) make sure the rule to pass keepalived +# multicast (224.0.0.18) traffic is in the table. +HA_IPTABLES_CHAIN="${OPENSHIFT_HA_IPTABLES_CHAIN:-""}" + +# Optional external check script that is run every HA_CHECK_INTERVAL seconds +# The script can test whatever is needed to verify the application is running. +# Must return 0 -- OK, or 1 -- fail +# This script is in addition to the default check that the port is listening. +# The script must be accessible from inside the keepalived pod +HA_CHECK_SCRIPT="${OPENSHIFT_HA_CHECK_SCRIPT:-""}" + +# Optional notify script is called when a state transition occurs +# Transition to MASTER, or to BACKUP, or to FAULT +# The parameters to the script are passed in by keepalived: +# $1 - "GROUP"|"INSTANCE" +# $2 - name of group or instance +# $3 - target state of transition ("MASTER"|"BACKUP"|"FAULT") +# The script must be accessible from inside the keepalived pod +HA_NOTIFY_SCRIPT="${OPENSHIFT_HA_NOTIFY_SCRIPT:-""}" + +# The check script is run every HA_CHECK_INTERVAL seconds. +# Default is 2 +HA_CHECK_INTERVAL="${OPENSHIFT_HA_CHECK_INTERVAL:-"2"}" + +# VRRP will preempt a lower priority machine when a higher priority one +# comes back online. You can change the preemption strategy to either: +# "nopreempt" - which allows the lower priority machine to maintain its +# 'MASTER' status. +# OR +# "preempt_delay 300" - waits 5 mins (in seconds) after startup to +# preempt lower priority MASTERs. +PREEMPTION="${OPENSHIFT_HA_PREEMPTION:-"preempt_delay 300"}" + + +# ======================================================================== +# Default settings - not currently exposed or overridden on OpenShift. +# ======================================================================== + +# If your environment doesn't support multicast, you can send VRRP adverts +# to a list of IPv{4,6} addresses using unicast. +# Example: +# UNICAST_PEERS="5.6.7.8,9.10.11.12,13.14.15.16" +UNICAST_PEERS="${OPENSHIFT_HA_UNICAST_PEERS:-""}" + + +# List of emails to send admin messages to. If the list of email ids is +# too long, you can use a DL (distribution list) ala: +# ADMIN_EMAILS=("ramr@redhat.com" "cops@acme.org") +ADMIN_EMAILS=(${OPENSHIFT_HA_ADMIN_EMAILS:-"root@localhost"}) + +# Email sender - the from address in the email headers. +EMAIL_FROM="ipfailover@openshift.local" + +# IP address of the SMTP server. +SMTP_SERVER="${OPENSHIFT_HA_SMTP_SERVER:-"127.0.0.1"}" + +# SMTP connect timeout (in seconds). +SMTP_CONNECT_TIMEOUT=30 + +# By default, the IP for binding vrrpd is the primary IP on the above +# specified interface. If you want to hide the location of vrrpd, you can +# specify a src_addr for multicast/unicast vrrp packets. +# MULTICAST_SOURCE_IPADDRESS="1.2.3.4" +# UNICAST_SOURCE_IPADDRESS="1.2.3.4" + diff --git a/lib/config-generators.sh b/lib/config-generators.sh new file mode 100755 index 0000000..8f58de4 --- /dev/null +++ b/lib/config-generators.sh @@ -0,0 +1,297 @@ +#!/bin/bash + +# TODO: This follows the initial demo pieces and uses a bash script to +# generate the keepalived config - rework this into a template +# similar to how it is done for the haproxy configuration. + +# Includes. +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" + + +# Constants. +readonly CHECK_SCRIPT_NAME="chk_${HA_CONFIG_NAME//-/_}" +readonly CHECK_INTERVAL_SECS="${HA_CHECK_INTERVAL}" +readonly VRRP_SLAVE_PRIORITY=42 + +readonly DEFAULT_PREEMPTION_STRATEGY="preempt_delay 300" + + +# +# Generate global config section. +# +# Example: +# generate_global_config arparp +# +function generate_global_config() { + local routername ; routername=$(scrub "$1") + + echo "global_defs {" + echo " notification_email {" + + for email in ${ADMIN_EMAILS[@]}; do + echo " $email" + done + + echo " }" + echo "" + echo " notification_email_from ${EMAIL_FROM:-"ipfailover@openshift.local"}" + echo " smtp_server ${SMTP_SERVER:-"127.0.0.1"}" + echo " smtp_connect_timeout ${SMTP_CONNECT_TIMEOUT:-"30"}" + echo " router_id ${routername}" + echo "}" +} + + +# +# Generate VRRP checker script configuration section. +# When a check script is provided use it instead of default script +# The default script is suppressed When port is 0 +# +# Example: +# generate_script_config +# generate_script_config "10.1.2.3" 8080 +# +function generate_script_config() { + local serviceip ; serviceip=${1:-"127.0.0.1"} + local port=${2:-80} + + echo "" + echo "vrrp_script ${CHECK_SCRIPT_NAME} {" + + if [[ -n "${HA_CHECK_SCRIPT}" ]]; then + echo " script \"${HA_CHECK_SCRIPT}\"" + else + if [[ "${port}" == "0" ]]; then + echo " script \"true\"" + else + echo " script \"/bin/bash -c ' /dev/null 2>&1 ; then + # Add the rule to the beginning of the chain. + echo " - adding iptables rule to $chain to access 224.0.0.18." + iptables -I ${chain} 1 -d 224.0.0.18/32 -j ACCEPT + fi + fi + + echo " - Generating and writing config to $KEEPALIVED_CONFIG" + generate_failover_config > "$KEEPALIVED_CONFIG" +} + + +function start_failover_services() { + echo " - Starting failover services ..." + + [ -f "$KEEPALIVED_DEFAULTS" ] && source "$KEEPALIVED_DEFAULTS" + + killall -9 /usr/sbin/keepalived &> /dev/null || : + /usr/sbin/keepalived $KEEPALIVED_OPTIONS -n --log-console & + local pid=$! + + trap "cleanup ${pid}" SIGHUP SIGINT SIGTERM + wait ${pid} +} + diff --git a/lib/utils.sh b/lib/utils.sh new file mode 100755 index 0000000..f350e6c --- /dev/null +++ b/lib/utils.sh @@ -0,0 +1,269 @@ +#!/bin/bash + + +# Constants. +LIB_DIR=$(dirname "${BASH_SOURCE[0]}") +VBOX_INTERFACES="enp0s3 enp0s8 eth1" + + +# +# Returns "scrubbed" name - removes characters that are not alphanumeric or +# underscore and replacing dashes with underscores. +# +# Examples: +# scrub "config\!@#@$%$^&*()-+=1_{}|[]\\:;'<>?,./ipfailover" +# # -> config_1_ipfailover +# +# scrub "ha-1" # -> ha_1 +# +function scrub() { + local val=$(echo "$1" | tr -dc '[:alnum:]\-_') + echo "${val//-/_}" +} + +# +# Tests if an IPv4 address is valid +# Echos true (0) if valid, false (1) if invalid +# +# Examples: +# validate_ipv4 192.0.2.3 +# # -> 0 +# +# validate_ipv4 192.0.3.4.0 +# # -> 1 +# +# validate_ipv4 192.0.2 +# # -> 1 +# +function validate_ipv4() { + local IPv4_GROUP="[0-2]?[0-9]{1,2}" + local IPv4_SHAPE="^(${IPv4_GROUP}\.){3,3}(${IPv4_GROUP})$" + local is_valid=0 + local i + + if [[ ${1} =~ ${IPv4_SHAPE} ]]; then + for i in $(echo ${1} | tr "." " "); do + if [ $i -gt 255 ]; then + is_valid=1 + fi + done + else + is_valid=1 + fi + return ${is_valid} +} + +# +# Tests if an IPv6 address is valid +# Returns true (0) if valid, false (1) if invalid +# +# Examples: +# validate_ipv6 2001:DB8:1:E32:FFFF:3:19:39FB +# # -> 0 +# +# validate_ipv6 2001:DB8::39FB +# # -> 0 +# +# validate_ipv6 2001::DB8::39FB +# # -> 1 +# +function validate_ipv6() { + local IPv6_GROUP="[[:xdigit:]]{1,4}" + local IPv6_SHAPE1="^(${IPv6_GROUP}:){7,7}(${IPv6_GROUP})$" + local IPv6_SHAPE2="^::(${IPv6_GROUP}:){0,6}(${IPv6_GROUP})$" + local IPv6_SHAPE3="^(${IPv6_GROUP}:){0,6}:(${IPv6_GROUP}:){0,6}(${IPv6_GROUP})$" + local VALID_SHAPE3="^(${IPv6_GROUP}:+){1,6}(${IPv6_GROUP})$" + + local is_valid=1 + + if [[ ${1} =~ ${IPv6_SHAPE1} ]]; then + is_valid=0 + fi + + if [[ ${1} =~ ${IPv6_SHAPE2} ]]; then + is_valid=0 + fi + + if [[ ${1} =~ ${IPv6_SHAPE3} ]]; then + if [[ ${1} =~ ${VALID_SHAPE3} ]]; then + is_valid=0 + fi + fi + return ${is_valid} +} + +# +# Expands list of IPv4 addresses. List elements can be an IP address +# range or an IP address. +# +# Examples: +# expand_ipv4_range "3.3.3.3-4" +# # -> 3.3.3.3 3.3.3.4 +# +# expand_ipv4_range "10.1.1.100-100" +# # -> 10.1.1.100 +# +# expand_ipv4_range "10.1.1.100" +# # -> 10.1.1.100 +# +function expand_ipv4_range() { + local expandedset=() + local ip1=$(echo "$1" | awk '{print $1}' FS='-') + local ip2=$(echo "$1" | awk '{print $2}' FS='-') + local n + + if [ -z "$ip2" ]; then + expandedset=(${expandedset[@]} "$ip1") + else + local base=$(echo "$ip1" | cut -f 1-3 -d '.') + local start=$(echo "$ip1" | awk '{print $NF}' FS='.') + local end=$(echo "$ip2" | awk '{print $NF}' FS='.') + for n in `seq $start $end`; do + expandedset=(${expandedset[@]} "${base}.$n") + done + fi + echo "${expandedset[@]}" +} + +# +# Expands list of IPv6 addresses. List elements can be an IP address +# range or an IP address. +# +# Examples: +# expand_ipv6_range "2001:DB8:1ABC::1F39-1F3B" +# # -> 2001:DB8:1ABC::1F39 2001:DB8:1ABC::1F3A 2001:DB8:1ABC::1F3B +# +function expand_ipv6_range() { + local expandedset=() + local ip1=$(echo "$1" | awk '{print $1}' FS='-') + local ip2=$(echo "$1" | awk '{print $2}' FS='-') + local n + if [ -z "$ip2" ]; then + expandedset=(${expandedset[@]} "$ip1") + else + local start=${ip1##*:} + local decstart=`echo "ibase=16; ${start}" | bc` + local base=${ip1%%${start}} + local decend=`echo "ibase=16; ${ip2}" | bc` + for n in `seq $decstart $decend`; do + end=`echo "obase=16; ${n}" | bc` + expandedset=(${expandedset[@]} "${base}${end}") + done + fi + echo "${expandedset[@]}" +} + +# +# Returns the IP address family (IPv4 or IPv6) +# Returns "4" or "6" respectively +# +# Examples: +# get_address_family "192.168.3.1" +# # -> 4 +# +# get_address_family "2001:DB8:1ABC::1F3A" +# # -> 6 +# +function get_address_family() { + if validate_ipv4 ${1}; then + return 4 + elif validate_ipv6 ${1}; then + return 6 + else + return 1 + fi +} + +# +# Expands list of virtual IP addresses. List elements can be an IP address +# range or an IP address and elements can be space or comma separated. +# +# Examples: +# expand_ip_ranges "1.1.1.1, 2.2.2.2,3.3.3.3-4 4.4.4.4" +# # -> 1.1.1.1 2.2.2.2 3.3.3.3 3.3.3.4 4.4.4.4 +# +# expand_ip_ranges "10.1.1.100-102 10.1.1.200-200 10.42.42.42" +# # -> 10.1.1.100 10.1.1.101 10.1.1.102 10.1.1.200 10.42.42.42 +# +function expand_ip_ranges() { + local vips=${1:-""} + local expandedset=() + local iprange + local newip + + for iprange in $(echo "$vips" | sed 's/[^0-9a-fA-F:\.,-]//g' | tr "," " "); do + local ip1=$(echo "$iprange" | awk '{print $1}' FS='-') + get_address_family ${ip1} + local family=$? + if [ ${family} == "4" ]; then + for newip in $(expand_ipv4_range ${iprange}); do + expandedset=(${expandedset[@]} ${newip}) + done + elif [ ${family} == "6" ]; then + for newip in $(expand_ipv6_range ${iprange}); do + expandedset=(${expandedset[@]} ${newip}) + done + fi + done + echo "${expandedset[@]}" +} + +# +# Generate base name for the VRRP instance. +# +# Examples: +# vrrp_instance_basename "arp" # -> arp_VIP +# +# vrrp_instance_basename "ha-1" # -> ha_1_VIP +# +function vrrp_instance_basename() { + echo "$(scrub "$1")_VIP" +} + + +# +# Generate VRRP instance name. +# +# Examples: +# generate_vrrp_instance_name arp 42 # -> arp_VIP_42 +# +# generate_vrrp_instance_name ha-1 # -> ha_1_VIP_0 +# +function generate_vrrp_instance_name() { + local iid=${2:-0} + echo "$(vrrp_instance_basename "$1")_${iid}" +} + + +# +# Returns the network device name to use for VRRP. +# +# Examples: +# get_network_device +# +# get_network_device "eth0" +# +function get_network_device() { + for dev in $1 ${VBOX_INTERFACES}; do + if ip addr show dev "$dev" &> /dev/null; then + echo "$dev" + return + fi + done + + ip route get 8.8.8.8 | awk '/dev/ { f=NR }; f && (NR-1 == f)' RS=" " +} + + +# +# Returns the IP address associated with a network device. +# +# Examples: +# get_device_ip_address +# +# get_device_ip_address "docker0" +# +function get_device_ip_address() { + local dev=${1:-"$(get_network_device)"} + ifconfig "$dev" | awk '/inet / { print $2 }' +} diff --git a/makefile.test b/makefile.test new file mode 100644 index 0000000..1c664ed --- /dev/null +++ b/makefile.test @@ -0,0 +1,11 @@ + +IMAGE_NAME="openshift/origin-keepalived-ipfailover" + + +all: image + +test: image + (cd tests && ./verify_failover_image.sh) + +image: + docker build -t $(IMAGE_NAME) . diff --git a/monitor.sh b/monitor.sh new file mode 100755 index 0000000..d7ef9dd --- /dev/null +++ b/monitor.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Includes. +source "$(dirname "${BASH_SOURCE[0]}")/lib/failover-functions.sh" + + +# +# main(): +# +setup_failover + +start_failover_services + +echo "`basename $0`: OpenShift IP Failover service terminated." + diff --git a/tests/echoserver.py b/tests/echoserver.py new file mode 100644 index 0000000..56293d3 --- /dev/null +++ b/tests/echoserver.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +""" Echo server - reply back with the received message. """ + +import os +import signal +import socket +import sys + + +def sigusr1_handler(signum, frame): + print 'signal %s received, exiting ...' % signum + sys.exit(0) + + +def setup(): + signal.signal(signal.SIGUSR1, sigusr1_handler) + + +def runserver(): + sock = socket.socket() + sock.bind(('0.0.0.0', int(os.environ.get('PORT', '12345')))) + sock.listen(10) + + while True: + c, raddr = sock.accept() + try: + d = c.recv(4096) + c.send(d if d else '') + finally: + c.close() + + +if "__main__" == __name__: + setup() + runserver() diff --git a/tests/verify_failover_image.sh b/tests/verify_failover_image.sh new file mode 100755 index 0000000..a757c15 --- /dev/null +++ b/tests/verify_failover_image.sh @@ -0,0 +1,90 @@ +#!/bin/bash -e + +# Constants. +readonly TEST_DIR=$(dirname "${BASH_SOURCE[0]}") +readonly FAILOVER_IMAGE="openshift/origin-keepalived-ipfailover" +readonly TEST_VIPS="10.0.2.100-102,2001:DB8:1ABC::1F39-1F3B" +readonly MONITOR_PORT="12345" + + +function stop_echo_server() { + local pid=$1 + if [ -z "$pid" ]; then + pid=$(ps -e -opid,args | grep echoserver.py | grep -v grep | awk '{print $1}') + fi + + # Send SIGUSR1 to the echo server to terminate it. + [ -n "$pid" ] && kill -s USR1 $pid +} + + +function start_echo_server() { + stop_echo_server + + export PORT=${MONITOR_PORT} + nohup python ${TEST_DIR}/echoserver.py &> /dev/null & + echo $! +} + + +function start_failover_container() { + local cfg="-e OPENSHIFT_HA_CONFIG_NAME="roto-r00ter"" + local vips="-e OPENSHIFT_HA_VIRTUAL_IPS="${TEST_VIPS}"" + local netif="-e OPENSHIFT_HA_NETWORK_INTERFACE="enp0s3"" + local port="-e OPENSHIFT_HA_MONITOR_PORT="${MONITOR_PORT}"" + # local unicast="-e export OPENSHIFT_HA_USE_UNICAST="true"" + # local unicastpeers="-e OPENSHIFT_HA_UNICAST_PEERS="127.0.0.1"" + local selector="-e OPENSHIFT_HA_SELECTOR=""" + local envopts="$cfg $vips $netif $port $unicast $unicastpeers $selector" + + docker run -dit --net=host --privileged=true \ + -v /lib/modules:/lib/modules $envopts $FAILOVER_IMAGE & + +} + + +function run_image_verification_test() { + echo " - starting echo server ..." + local pid=$(start_echo_server) + echo " - started echo server pid=$pid ..." + + # On interrupt, cleanup - stop echo server. + trap "stop_echo_server $pid" INT + + local cname=$(start_failover_container) + echo " - started docker container $cname ..." + + # Wait a bit for all the services to startup. + sleep 10 + + # Check container is up and has keepalived processes. + local cmd="ps -ef | grep '/usr/sbin/keepalived' | grep -v grep | wc -l" + local numprocs=$(echo "$cmd" | docker exec -i $cname /bin/bash) + + # Stop echo server. + stop_echo_server $pid + + if [[ -n "$numprocs" && $numprocs -gt 0 ]]; then + # Success - print info and kill the container. + echo " - There are $numprocs keepalived processes running" + echo " - Cleaning up docker containers ..." + docker rm -f $cname + echo " - All tests PASSED." + return 0 + fi + + # Failure - print info and dump logs (keep the docker container around + # for debugging). + echo " - There are $numprocs keepalived processes running" + echo " - logs from container $cname:" + docker logs $cname || : + echo " - Test FAILED." + exit 1 +} + + +# +# main(): +# +run_image_verification_test +