Openshift-UPI: Keepalived to rescue failover

kind: Pod
apiVersion: v1
metadata:
  name: keepalived
  namespace: openshift-openstack-infra
  creationTimestamp:
  deletionGracePeriodSeconds: 65
  labels:
    app: openstack-infra-vrrp
spec:
  volumes:
  - name: resource-dir
    hostPath:
      path: "/etc/kubernetes/static-pod-resources/keepalived"
  - name: script-dir
    hostPath:
      path: "/etc/kubernetes/static-pod-resources/keepalived/scripts"
  - name: kubeconfig
    hostPath:
      path: "/etc/kubernetes"
  - name: kubeconfigvarlib
    hostPath:
      path: "/var/lib/kubelet"
  - name: conf-dir
    hostPath:
      path: "/etc/keepalived"
  - name: run-dir
    empty-dir: {}
  - name: chroot-host
    hostPath:
      path: "/"
  initContainers:
  - name: render-config-keepalived
    image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f0f7c54f0f2ecd38fdf2667651f95153a589bd7fe4605f0f96a97899576a08
    command:
    - runtimecfg
    - render
    - "/etc/kubernetes/kubeconfig"
    - "--api-vip"
    - "172.21.104.25"
    - "--ingress-vip"
    - "172.21.104.26"
    - "/config"
    - "--out-dir"
    - "/etc/keepalived"
    resources: {}
    volumeMounts:
    - name: kubeconfig
      mountPath: "/etc/kubernetes"
    - name: script-dir
      mountPath: "/config"
    - name: conf-dir
      mountPath: "/etc/keepalived"
    imagePullPolicy: IfNotPresent
  containers:
  - name: keepalived
    securityContext:
      privileged: true
    image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8c872154e89c7c361c882bcd6acff3ff97464e7f1a0311c631e9a939085e9934
    env:
      - name: NSS_SDB_USE_CACHE
        value: "no"
    command:
    - /bin/bash
    - -c
    - |
      #/bin/bash
      reload_keepalived()
      {
        if pid=$(pgrep -o keepalived); then
            kill -s SIGHUP "$pid"
        else
            /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console &
        fi
      }

      msg_handler()
      {
        while read -r line; do
          echo "The client sent: $line" >&2
          # currently only 'reload' msg is supported
          if [ "$line" = reload ]; then
              reload_keepalived
          fi
        done
      }

      set -ex
      declare -r keepalived_sock="/var/run/keepalived/keepalived.sock"
      export -f msg_handler
      export -f reload_keepalived
      if [ -s "/etc/keepalived/keepalived.conf" ]; then
          /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console &
      fi

      rm -f "$keepalived_sock"
      socat UNIX-LISTEN:${keepalived_sock},fork system:'bash -c msg_handler'
    resources:
      requests:
        cpu: 100m
        memory: 200Mi
    volumeMounts:
    - name: conf-dir
      mountPath: "/etc/keepalived"
    - name: run-dir
      mountPath: "/var/run/keepalived"
    livenessProbe:
      exec:
        command:
        - /bin/bash
        - -c
        - |
          kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data
      initialDelaySeconds: 20
    terminationMessagePolicy: FallbackToLogsOnError
    imagePullPolicy: IfNotPresent
  - name: keepalived-monitor
    securityContext:
      privileged: true
    image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f0f7c54f0f2ecd38fdf2667651f95153a589bd7fe4605f0f96a97899576a08
    env:
      - name: ENABLE_UNICAST
        value: "no"
      - name: IS_BOOTSTRAP
        value: "no"
    command:
    - dynkeepalived
    - "/var/lib/kubelet/kubeconfig"
    - "/config/keepalived.conf.tmpl"
    - "/etc/keepalived/keepalived.conf"
    - "--api-vip"
    - "172.21.104.25"
    - "--ingress-vip"
    - "172.21.104.26"
    resources:
      requests:
        cpu: 100m
        memory: 200Mi
    volumeMounts:
    - name: resource-dir
      mountPath: "/config"
    - name: kubeconfigvarlib
      mountPath: "/var/lib/kubelet"
    - name: conf-dir
      mountPath: "/etc/keepalived"
    - name: run-dir
      mountPath: "/var/run/keepalived"
    - name: chroot-host
      mountPath: "/host"
    imagePullPolicy: IfNotPresent
  hostNetwork: true
  tolerations:
  - operator: Exists
  priorityClassName: system-node-critical
status: {}

global_defs {
    enable_script_security
    script_user root
}

# These are separate checks to provide the following behavior:
# If the loadbalanced endpoint is responding then all is well regardless
# of what the local api status is. Both checks will return success and
# we'll have the maximum priority. This means as long as there is a node
# with a functional loadbalancer it will get the VIP.
# If all of the loadbalancers go down but the local api is still running,
# the _both check will still succeed and allow any node with a functional
# api to take the VIP. This isn't preferred because it means all api
# traffic will go through one node, but at least it keeps the api available.
vrrp_script chk_ocp_lb {
    script "/usr/bin/timeout 1.9 /etc/keepalived/chk_ocp_script.sh"
    interval 2
    weight 20
    rise 3
    fall 2
}

vrrp_script chk_ocp_both {
    script "/usr/bin/timeout 1.9 /etc/keepalived/chk_ocp_script_both.sh"
    interval 2
    # Use a smaller weight for this check so it won't trigger the move from
    # bootstrap to master by itself.
    weight 5
    rise 3
    fall 2
}

# TODO: Improve this check. The port is assumed to be alive.
# Need to assess what is the ramification if the port is not there.
vrrp_script chk_ingress {
    script "/usr/bin/timeout 0.9 /usr/bin/curl -o /dev/null -Lfs http://localhost:1936/healthz/ready"
    interval 1
    weight 50
}

{{$nonVirtualIP := .NonVirtualIP}}

vrrp_instance {{ .Cluster.Name }}_API {
    state BACKUP
    interface {{ .VRRPInterface }}
    virtual_router_id {{ .Cluster.APIVirtualRouterID }}
    priority 40
    advert_int 1
    {{if .EnableUnicast}}
    unicast_src_ip {{.NonVirtualIP}}
    unicast_peer {
        {{ .BootstrapIP }}
        {{range .LBConfig.Backends}}
        {{if ne $nonVirtualIP .Address}}{{.Address}}{{end}}
        {{end}}
    }
    {{end}}
    authentication {
        auth_type PASS
        auth_pass {{ .Cluster.Name }}_api_vip
    }
    virtual_ipaddress {
        {{ .Cluster.APIVIP }}/{{ .Cluster.VIPNetmask }}
    }
    track_script {
        chk_ocp_lb
        chk_ocp_both
    }
}


vrrp_instance {{ .Cluster.Name }}_INGRESS {
    state BACKUP
    interface {{ .VRRPInterface }}
    virtual_router_id {{ .Cluster.IngressVirtualRouterID }}
    priority 40
    advert_int 1
    {{if .EnableUnicast}}
    unicast_src_ip {{.NonVirtualIP}}
    unicast_peer {
        {{range .IngressConfig.Peers}}
        {{if ne $nonVirtualIP .}}{{.}}{{end}}
        {{end}}
    }
    {{end}}
    authentication {
        auth_type PASS
        auth_pass {{ .Cluster.Name }}_ingress_vip
    }
    virtual_ipaddress {
        {{ .Cluster.IngressVIP }}/{{ .Cluster.VIPNetmask }}
    }
    track_script {
        chk_ingress
    }
}