Calico BGP route reflectors and IP in IP

# RHEL 7 / CentOS 7
# DISRUPTIVE - running "kubectl drain" on existing nodes to make sure no workloads are running on the nodes.
# DISRUPTIVE - configure the BGPPeer resources before disabling the node-to-node mesh to avoid pod networking breakage.
# Reference - https://www.tigera.io/blog/configuring-route-reflectors-in-calico/

# install calicoctl on all nodes to check the calico BGP status
sudo curl -L https://github.com/projectcalico/calico/releases/latest/download/calicoctl-linux-amd64 -o calicoctl
sudo chmod +x calicoctl
sudo cp -p calicoctl /usr/bin
sudo calicoctl node status

# [optional] make sure no workloads are running on the nodes -> drain the nodes
sudo kubectl get nodes -owide
sudo kubectl drain --ignore-daemonsets <node name>

# on k8s-master-1, create Calico BGP configuration
sudo calicoctl apply -f - --allow-version-mismatch <<EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: true
  asNumber: 64567
EOF

# verify
sudo calicoctl get bgpconfiguration --allow-version-mismatch

# choose k8s-master-1, 2, and 3 as route reflectors
# k8s-master-1
sudo kubectl annotate node k8s-master-1 projectcalico.org/RouteReflectorClusterID=1.0.0.1
sudo kubectl label node k8s-master-1 route-reflector=true
# k8s-master-2
sudo kubectl annotate node k8s-master-2 projectcalico.org/RouteReflectorClusterID=1.0.0.1
sudo kubectl label node k8s-master-2 route-reflector=true
# k8s-master-3
sudo kubectl annotate node k8s-master-3 projectcalico.org/RouteReflectorClusterID=1.0.0.1
sudo kubectl label node k8s-master-3 route-reflector=true

# on master nodes, add iBGP peering (rr-to-rr-peer)
sudo calicoctl apply -f - --allow-version-mismatch <<EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: rr-to-rr-peer
spec:
  nodeSelector: has(route-reflector)
  peerSelector: has(route-reflector)
EOF

# on worker nodes, add iBGP peering (node-peer-to-rr)
sudo calicoctl apply -f - --allow-version-mismatch <<EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: node-peer-to-rr
spec:
  nodeSelector: !has(route-reflector)
  peerSelector: has(route-reflector)
EOF

# verify
sudo calicoctl node status
sudo calicoctl get bgppeer --allow-version-mismatch

# on k8s-master-1, disable the BGP node-to-node mesh for the cluster.
sudo calicoctl patch bgpconfiguration default -p '{"spec": {"nodeToNodeMeshEnabled": false}}' --allow-version-mismatch

# verify
sudo calicoctl get bgpconfiguration --allow-version-mismatch
sudo calicoctl node status
sudo calicoctl get bgppeer --allow-version-mismatch

# on k8s-master-1, edit overlay networking - ippools
sudo kubectl edit ippools
# update the contents
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
  name: default-ipv4-ippool
spec:
  cidr: 192.160.0.0/16
  ipipMode: CrossSubnet
  natOutgoing: true
  nodeSelector: all()
  vxlanMode: Never

# verify 
sudo calicoctl get ippool -o wide --allow-version-mismatch

# [optional] resume scheduling new pods onto the node
sudo kubectl uncordon <node name>