Highly Available Stacked Kubernetes cluster requires:

  • 3 virtual machines to provision nodes with control plane and etcd database
  • 1-3 worker nodes
  • each VM requires
    • 4 CPU cores, 4 GB RAM virtual machine with DHCP reservation

Configure Parallels Network

First reduce DHCP IP range to be able to set VIP without causing IP conflict with DHCP assigned IPs

prlsrvctl net set Shared --ip-scope-end $(prlsrvctl net info Shared | grep DHCPv4 -A3 | tail -n 1 | awk -F ": " '{print $2}' | awk -F"." '{print $1"."$2"."$3"."199}')

Provision nodes for control Plane and etcd

  • Create 3 control-plane VMs from k8s-vm-template. One of the way of achieving high availability is to use keepalived in combination with haproxy load balancer. keepalived relies on Linux Virtual Server (IPVS) kernel module providing Layer 4 load balancing. A second virtual IP will be assigned to one of the VMs based on election process. haproxy

    for i in $(seq 1 3)
    do
      echo "###################################"
      echo "### configuring orion-control$i ###"
      echo "###################################"
      prlctl create orion-control$i --ostemplate k8s-vm-template
      # configure VM cpu, ram and set different mac addresses
      prlctl set orion-control$i --cpus=2 --memsize=4G --startup-view headless --on-window-close keep-running --sync-vm-hostname on --device-set net0 --mac 00:00:00:00:02:0$i
      prlctl start orion-control$i
      # wait for start
      printf "%s" "waiting for orion-control$i ..."
      while ! timeout 0.2 prlctl status orion-control$i | grep running &> /dev/null; do printf "%s" "."; done
      # configure DHCP client to use mac address for IP assignments
      export IPLINK=$(prlctl exec orion-control$i "ip -br l" | awk '$1 !~ "lo" {print $1}')
      echo "IPLINK: $IPLINK"
      if [ -z "${IPLINK}" ]; then
        echo IPLINK is empty. rerun script
        break
      fi
      # configure network
      prlctl exec orion-control$i netplan set ethernets.$IPLINK.dhcp-identifier=mac
      prlctl exec orion-control$i netplan apply
      prlctl exec orion-control$i ip a
      # set hostname in case parallels failed to do so
      prlctl exec orion-control$i hostnamectl set-hostname orion-control$i
      prlctl exec orion-control$i systemctl restart systemd-hostnamed
      # pull images
      prlctl exec orion-control$i kubeadm config images pull
      # install keepalived and haproxy
      prlctl exec orion-control$i apt install haproxy keepalived -y
      export CONTROL$iIP=$(prlctl list -i orion-control$i -f -j | jq -r '.[0].Network.ipAddresses[] | select(.type=="ipv4") | .ip')
    done
    

Enable High Availability

  • Configure keepalived with exported VIP

    export VIP=$(prlsrvctl net info Shared | grep DHCPv4 -A3 | tail -n 1 | awk -F ": " '{print $2}' | awk -F"." '{print $1"."$2"."$3"."200}')
    echo "configuring orion-control1 as a default master with VIP: $VIP"
    prlctl exec orion-control1 "cat << EOF | tee /etc/keepalived/keepalived.conf
    vrrp_instance VI_1 {
      state MASTER
      interface $IPLINK
      virtual_router_id 51
      priority 255
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass 12345
      }
      virtual_ipaddress {
          $VIP/24
      }
    }
    EOF
    "
    prlctl exec orion-control1 "systemctl enable keepalived --now"
    
    # configuring orion-control2
    echo "configuring orion-control2 as the first backup"
    prlctl exec orion-control2 "cat << EOF | tee /etc/keepalived/keepalived.conf
    vrrp_instance VI_1 {
      state BACKUP
      interface $IPLINK
      virtual_router_id 51
      priority 254
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass 12345
      }
      virtual_ipaddress {
          $VIP/24
      }
    }
    EOF
    "
    prlctl exec orion-control2 "systemctl enable keepalived --now"
    
    # configuring orion-control3
    echo "configuring orion-control3 as the second backup"
    prlctl exec orion-control3 "cat << EOF | tee /etc/keepalived/keepalived.conf
    vrrp_instance VI_1 {
      state BACKUP
      interface $IPLINK
      virtual_router_id 51
      priority 253
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass 12345
      }
      virtual_ipaddress {
          $VIP/24
      }
    }
    EOF
    "
    prlctl exec orion-control3 "systemctl enable keepalived --now"
    
  • Configure haproxy round robin load balancer for all three control plane nodes

    for i in $(seq 1 3)
    do
      echo "configuring haproxy in orion-control$i"
      prlctl exec orion-control$i "cat << EOF | tee /etc/haproxy/haproxy.cfg
    defaults
      mode http
      timeout client 10s
      timeout connect 5s
      timeout server 10s 
      timeout http-request 10s
    frontend myfrontend
      bind 127.0.0.1:8443
    backend apiserver
      option httpchk GET /healthz
      http-check expect status 200
      mode tcp
      option ssl-hello-chk
      balance     roundrobin
        server orion-control1 $CONTROL1IP:6443 check
        server orion-control2 $CONTROL2IP:6443 check
        server orion-control3 $CONTROL3IP:6443 check
    EOF
      "
      prlctl exec orion-control$i "sudo systemctl enable haproxy --now"
    done
    

Initialize kubernetes control plane nodes

  • Init kubernetes on control-orion1

    # init kubernetes cluster on master node `orion-control1`
    prlctl exec orion-control1 kubeadm init --control-plane-endpoint $VIP:6443 --upload-certs > /tmp/kubeadm
    # enable calico
    prlctl exec orion-control1 --user cka kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/calico.yaml
    # configure `kubectl` client on `orion-control1`
    prlctl exec orion-control1 --user cka mkdir -p /home/cka/.kube
    prlctl exec orion-control1 cp -i /etc/kubernetes/admin.conf /home/cka/.kube/config
    prlctl exec orion-control1 chown 1000:1000 /home/cka/.kube/config
    
  • Join other control plane nodes

    for i in $(seq 2 3)
    do
      prlctl exec orion-control$i "$(cat /tmp/kubeadm | grep "You can now join any number of control" -A4 | tail -n 3)"
      prlctl exec orion-control$i --user cka mkdir -p /home/cka/.kube
      prlctl exec orion-control$i cp -i /etc/kubernetes/admin.conf /home/cka/.kube/config
      prlctl exec orion-control$i chown 1000:1000 /home/cka/.kube/config
    done
    

Provision Worker Nodes

  • Create three worker nodes

    for i in $(seq 1 3)
    do
      echo "###################################"
      echo "### configuring orion-worker$i ###"
      echo "###################################"
      prlctl create orion-worker$i --ostemplate k8s-vm-template
      # configure VM cpu, ram and set different mac addresses
      prlctl set orion-worker$i --cpus=4 --memsize=4G --startup-view headless --on-window-close keep-running --sync-vm-hostname on --device-set net0 --mac 00:00:00:00:03:0$i
      prlctl start orion-worker$i
      # wait for start
      printf "%s" "waiting for orion-worker$i ..."
      while ! timeout 0.2 prlctl status orion-worker$i | grep running &> /dev/null; do printf "%s" "."; done
      # configure DHCP client to use mac address for IP assignments
      export IPLINK=$(prlctl exec orion-worker$i "ip -br l" | awk '$1 !~ "lo" {print $1}')
      echo "IPLINK: $IPLINK"
      if [ -z "${IPLINK}" ]; then
        echo IPLINK is empty. rerun script
        break
      fi
      # configure network
      prlctl exec orion-worker$i netplan set ethernets.$IPLINK.dhcp-identifier=mac
      prlctl exec orion-worker$i netplan apply
      prlctl exec orion-worker$i ip a
      # set hostname in case parallels failed to do so
      prlctl exec orion-worker$i hostnamectl set-hostname orion-worker$i
      prlctl exec orion-worker$i systemctl restart systemd-hostnamed
    done
    

Add worker nodes to the cluster

  • Join worker nodes

    for i in $(seq 1 3)
    do
      prlctl exec orion-worker$i "$(cat /tmp/kubeadm | grep "you can join any number of worker" -A3 | tail -n 2)"
    done