Self-Managed Kubernetes Cluster on AWS (with Traefik & MetalLB)

Architecture Overview:

Disable Swap & Configure Kernel (Control + Worker Nodes)

sudo swapoff -a
sudo sed -i '/ swap / s/^/#/' /etc/fstab
 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
 
sudo modprobe overlay
sudo modprobe br_netfilter
 
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
 
sudo sysctl --system

Install containerd (Control + Worker Nodes)

sudo apt update && sudo apt install -y ca-certificates curl gnupg lsb-release
 
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
 
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \
https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
sudo apt update && sudo apt install -y containerd.io

Configure containerd (Control + Worker Nodes)

containerd config default | sudo tee /etc/containerd/config.toml

Edit /etc/containerd/config.toml and ensure:

[plugins."io.containerd.grpc.v1.cri"]
  sandbox_image = "registry.k8s.io/pause:3.10"
 
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true

sudo systemctl restart containerd
sudo systemctl enable containerd

Install Kubernetes binaries (Control + Worker Nodes)

sudo apt install -y apt-transport-https
 
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | \
  sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/kubernetes.gpg
 
echo "deb [signed-by=/etc/apt/trusted.gpg.d/kubernetes.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" | \
  sudo tee /etc/apt/sources.list.d/kubernetes.list
 
sudo apt update
sudo apt install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

Install Helm (Control Node)

curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
sudo apt-get update
sudo apt-get install helm

Initialize Control Plane (Control Node)

sudo kubeadm init --pod-network-cidr=172.16.0.0/16

Set up kubectl:

mkdir -p ~/.kube
sudo cp /etc/kubernetes/admin.conf ~/.kube/config
sudo chown $(id -u):$(id -g) ~/.kube/config

Install Calico CNI (Control Node)

kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/calico.yaml

Install MetalLB (Control Node)

helm repo add metallb https://metallb.github.io/metallb
helm repo update
helm install metallb metallb/metallb --namespace metallb-system --create-namespace

Configure MetalLB (10.0.128.240 is a secondary IP added to your control-plane EC2 instance)

# metallb-config.yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: control-node-pool
  namespace: metallb-system
spec:
  addresses:
    - 10.0.128.240/32
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: advertise
  namespace: metallb-system
spec:
  ipAddressPools:
    - control-node-pool
  nodeSelectors:
    - matchLabels:
        kubernetes.io/hostname: ip-10-0-128-6

Apply the configuration:

kubectl apply -f metallb-config.yaml

Install Traefik Ingress (Control Node)

# traefik-values.yaml
service:
  type: LoadBalancer
  loadBalancerIP: 10.0.128.240
 
additionalArguments:
  - "--ping=true"
  - "--ping.entrypoint=traefik"
  - "--api.dashboard=true"
 
ingressClass:
  enabled: true
  isDefaultClass: true
  name: traefik
 
nodeSelector:
  kubernetes.io/hostname: ip-10-0-128-6
 
tolerations:
  - key: "node-role.kubernetes.io/control-plane"
    operator: "Exists"
    effect: "NoSchedule"

helm repo add traefik https://traefik.github.io/charts
helm repo update
helm install traefik traefik/traefik \
  --namespace kube-system \
  -f traefik-values.yaml

Deploy whoami Demo App (Control Node)

# whoami.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: whoami
spec:
  replicas: 1
  selector:
    matchLabels:
      app: whoami
  template:
    metadata:
      labels:
        app: whoami
    spec:
      containers:
        - name: whoami
          image: traefik/whoami
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: whoami
spec:
  selector:
    app: whoami
  ports:
    - port: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: whoami
spec:
  rules:
    - host: whoami.maksonlee.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: whoami
                port:
                  number: 80

kubectl apply -f whoami.yaml

Leave a Comment Cancel Reply