Highly Available Stacked Kubernetes cluster requires:
- 3 virtual machines to provision nodes with control plane and
etcd
database - 1-3 worker nodes
- each VM requires
- 4 CPU cores, 4 GB RAM virtual machine with DHCP reservation
Configure Parallels Network
First reduce DHCP IP range to be able to set VIP without causing IP conflict with DHCP assigned IPs
prlsrvctl net set Shared --ip-scope-end $(prlsrvctl net info Shared | grep DHCPv4 -A3 | tail -n 1 | awk -F ": " '{print $2}' | awk -F"." '{print $1"."$2"."$3"."199}')
Provision nodes for control Plane and etcd
-
Create 3 control-plane VMs from
k8s-vm-template
. One of the way of achieving high availability is to usekeepalived
in combination withhaproxy
load balancer.keepalived
relies on Linux Virtual Server (IPVS) kernel module providing Layer 4 load balancing. A second virtual IP will be assigned to one of the VMs based on election process.haproxy
for i in $(seq 1 3) do echo "###################################" echo "### configuring orion-control$i ###" echo "###################################" prlctl create orion-control$i --ostemplate k8s-vm-template # configure VM cpu, ram and set different mac addresses prlctl set orion-control$i --cpus=2 --memsize=4G --startup-view headless --on-window-close keep-running --sync-vm-hostname on --device-set net0 --mac 00:00:00:00:02:0$i prlctl start orion-control$i # wait for start printf "%s" "waiting for orion-control$i ..." while ! timeout 0.2 prlctl status orion-control$i | grep running &> /dev/null; do printf "%s" "."; done # configure DHCP client to use mac address for IP assignments export IPLINK=$(prlctl exec orion-control$i "ip -br l" | awk '$1 !~ "lo" {print $1}') echo "IPLINK: $IPLINK" if [ -z "${IPLINK}" ]; then echo IPLINK is empty. rerun script break fi # configure network prlctl exec orion-control$i netplan set ethernets.$IPLINK.dhcp-identifier=mac prlctl exec orion-control$i netplan apply prlctl exec orion-control$i ip a # set hostname in case parallels failed to do so prlctl exec orion-control$i hostnamectl set-hostname orion-control$i prlctl exec orion-control$i systemctl restart systemd-hostnamed # pull images prlctl exec orion-control$i kubeadm config images pull # install keepalived and haproxy prlctl exec orion-control$i apt install haproxy keepalived -y export CONTROL$iIP=$(prlctl list -i orion-control$i -f -j | jq -r '.[0].Network.ipAddresses[] | select(.type=="ipv4") | .ip') done
Enable High Availability
-
Configure
keepalived
with exported VIPexport VIP=$(prlsrvctl net info Shared | grep DHCPv4 -A3 | tail -n 1 | awk -F ": " '{print $2}' | awk -F"." '{print $1"."$2"."$3"."200}') echo "configuring orion-control1 as a default master with VIP: $VIP" prlctl exec orion-control1 "cat << EOF | tee /etc/keepalived/keepalived.conf vrrp_instance VI_1 { state MASTER interface $IPLINK virtual_router_id 51 priority 255 advert_int 1 authentication { auth_type PASS auth_pass 12345 } virtual_ipaddress { $VIP/24 } } EOF " prlctl exec orion-control1 "systemctl enable keepalived --now" # configuring orion-control2 echo "configuring orion-control2 as the first backup" prlctl exec orion-control2 "cat << EOF | tee /etc/keepalived/keepalived.conf vrrp_instance VI_1 { state BACKUP interface $IPLINK virtual_router_id 51 priority 254 advert_int 1 authentication { auth_type PASS auth_pass 12345 } virtual_ipaddress { $VIP/24 } } EOF " prlctl exec orion-control2 "systemctl enable keepalived --now" # configuring orion-control3 echo "configuring orion-control3 as the second backup" prlctl exec orion-control3 "cat << EOF | tee /etc/keepalived/keepalived.conf vrrp_instance VI_1 { state BACKUP interface $IPLINK virtual_router_id 51 priority 253 advert_int 1 authentication { auth_type PASS auth_pass 12345 } virtual_ipaddress { $VIP/24 } } EOF " prlctl exec orion-control3 "systemctl enable keepalived --now"
-
Configure
haproxy
round robin load balancer for all three control plane nodesfor i in $(seq 1 3) do echo "configuring haproxy in orion-control$i" prlctl exec orion-control$i "cat << EOF | tee /etc/haproxy/haproxy.cfg defaults mode http timeout client 10s timeout connect 5s timeout server 10s timeout http-request 10s frontend myfrontend bind 127.0.0.1:8443 backend apiserver option httpchk GET /healthz http-check expect status 200 mode tcp option ssl-hello-chk balance roundrobin server orion-control1 $CONTROL1IP:6443 check server orion-control2 $CONTROL2IP:6443 check server orion-control3 $CONTROL3IP:6443 check EOF " prlctl exec orion-control$i "sudo systemctl enable haproxy --now" done
Initialize kubernetes control plane nodes
-
Init kubernetes on
control-orion1
# init kubernetes cluster on master node `orion-control1` prlctl exec orion-control1 kubeadm init --control-plane-endpoint $VIP:6443 --upload-certs > /tmp/kubeadm # enable calico prlctl exec orion-control1 --user cka kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/calico.yaml # configure `kubectl` client on `orion-control1` prlctl exec orion-control1 --user cka mkdir -p /home/cka/.kube prlctl exec orion-control1 cp -i /etc/kubernetes/admin.conf /home/cka/.kube/config prlctl exec orion-control1 chown 1000:1000 /home/cka/.kube/config
-
Join other control plane nodes
for i in $(seq 2 3) do prlctl exec orion-control$i "$(cat /tmp/kubeadm | grep "You can now join any number of control" -A4 | tail -n 3)" prlctl exec orion-control$i --user cka mkdir -p /home/cka/.kube prlctl exec orion-control$i cp -i /etc/kubernetes/admin.conf /home/cka/.kube/config prlctl exec orion-control$i chown 1000:1000 /home/cka/.kube/config done
Provision Worker Nodes
-
Create three worker nodes
for i in $(seq 1 3) do echo "###################################" echo "### configuring orion-worker$i ###" echo "###################################" prlctl create orion-worker$i --ostemplate k8s-vm-template # configure VM cpu, ram and set different mac addresses prlctl set orion-worker$i --cpus=4 --memsize=4G --startup-view headless --on-window-close keep-running --sync-vm-hostname on --device-set net0 --mac 00:00:00:00:03:0$i prlctl start orion-worker$i # wait for start printf "%s" "waiting for orion-worker$i ..." while ! timeout 0.2 prlctl status orion-worker$i | grep running &> /dev/null; do printf "%s" "."; done # configure DHCP client to use mac address for IP assignments export IPLINK=$(prlctl exec orion-worker$i "ip -br l" | awk '$1 !~ "lo" {print $1}') echo "IPLINK: $IPLINK" if [ -z "${IPLINK}" ]; then echo IPLINK is empty. rerun script break fi # configure network prlctl exec orion-worker$i netplan set ethernets.$IPLINK.dhcp-identifier=mac prlctl exec orion-worker$i netplan apply prlctl exec orion-worker$i ip a # set hostname in case parallels failed to do so prlctl exec orion-worker$i hostnamectl set-hostname orion-worker$i prlctl exec orion-worker$i systemctl restart systemd-hostnamed done
Add worker nodes to the cluster
-
Join worker nodes
for i in $(seq 1 3) do prlctl exec orion-worker$i "$(cat /tmp/kubeadm | grep "you can join any number of worker" -A3 | tail -n 2)" done