Install k3s with metallb

1
2
3
4
5
6
7

# map 443 and 80 ports to ethernet on node
k3d cluster create mycluster --agents 3 -p "443:443@loadbalancer" -p "80:80@loadbalancer" -p "8000:8000@loadbalancer" --wait
# k3d cluster create mycluster --agents 3 --k3s-arg "--disable=traefik@server:0" --wait
k cluster-info                                 
  Kubernetes control plane is running at https://0.0.0.0:39973
  CoreDNS is running at https://0.0.0.0:39973/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
  Metrics-server is running at https://0.0.0.0:39973/api/v1/namespaces/kube-system/services/https:metrics-server:https/proxy

1
2
3
4
5
6

k get nodes -o wide
  NAME                     STATUS   ROLES                  AGE   VERSION        INTERNAL-IP   EXTERNAL-IP   OS-IMAGE   KERNEL-VERSION      CONTAINER-RUNTIME
  k3d-mycluster-server-0   Ready    control-plane,master   42s   v1.26.4+k3s1   172.18.0.2    <none>        K3s dev    5.19.0-46-generic   containerd://1.6.19-k3s1
  k3d-mycluster-agent-0    Ready    <none>                 38s   v1.26.4+k3s1   172.18.0.4    <none>        K3s dev    5.19.0-46-generic   containerd://1.6.19-k3s1
  k3d-mycluster-agent-1    Ready    <none>                 37s   v1.26.4+k3s1   172.18.0.3    <none>        K3s dev    5.19.0-46-generic   containerd://1.6.19-k3s1
  k3d-mycluster-agent-2    Ready    <none>                 37s   v1.26.4+k3s1   172.18.0.5    <none>        K3s dev    5.19.0-46-generic   containerd://1.6.19-k3s1

1
2
3

helm repo add metallb https://metallb.github.io/metallb
helm search repo metallb
helm install --generate-name --namespace metallb-system --create-namespace metallb/metallb --version 0.14.5

1
2

docker network inspect k3d-mycluster | jq '.[0].IPAM.Config[0].Subnet' |  tr -d '"'
172.20.0.0/16

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

cat << 'EOF' | kubectl apply -f -
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: default-pool
  namespace: metallb-system
spec:
  addresses:
  - 172.20.7.7-172.20.7.77
  - 172.20.7.6-172.20.7.6
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: default
  namespace: metallb-system
spec:
  ipAddressPools:
  - default-pool
EOF

1
2

k rollout restart deployments.apps --namespace metallb-system metallb-1690485833-controller
k rollout restart daemonset --namespace metallb-system metallb-1690485833-speaker

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# create a deployment (i.e. game 2048)
k create deployment game2048 --image=alexwhen/docker-2048

# expose the deployments using a LoadBalancer
k expose deployment game2048 --port=80 --type=LoadBalancer

# obtain the ingress external ip
external_ip=$(k get svc game2048 -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

# test the loadbalancer external ip
curl $external_ip

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# create a deployment (i.e. game 2048)
k create deployment game2048-2 --image=alexwhen/docker-2048

# expose the deployments using a LoadBalancer
# --target-port='': Name or number for the port on the container that the service should direct traffic to. Optional.
k expose deployment game2048-2 --port=80 --target-port 80 --type=ClusterIP
# or
# k create service clusterip game2048-2 --tcp 80:80

# create ingress, note we will get 404 from containred nginx (game-2048) with aliases: some*, v1, v2 because them doesn't exist in our container. It needs to find approach with https://doc.traefik.io/traefik/v1.7/configuration/backends/kubernetes/#annotations, like redirect-regex and redirect-replacement

# Rule in format host/path=service:port[,tls=secretname].
# Paths containing the leading character '*' are considered pathType=Prefix. tls argument is optional.
k create ingress traefik-app \
  --rule='check.my.awesome.ingress.com/*=game2048-2:80' \
  --rule='check.my.awesome.ingress.com/some*=game2048-2:80' \
  --rule=check.my.awesome.ingress.com/v1=game2048-2:80 \
  --rule=check.my.awesome.ingress.com/v2=game2048-2:80 \
  --class="traefik" \
  --annotation traefik.frontend.passHostHeader="true" \
  --annotation traefik.backend.loadbalancer.sticky="true" \
  --dry-run=client -o yaml | k apply -f-

k create ingress traefik-app-test \
  --rule='check.my.awesome.ingress.com/some*=nginx:80' \
  --rule=check.my.awesome.ingress.com/v1=nginx:80 \
  --rule=check.my.awesome.ingress.com/v2=nginx:80 \
  --annotation traefik.ingress.kubernetes.io/redirect-regex='^http://localhost/(.*)' \
  --annotation traefik.ingress.kubernetes.io/redirect-replacement='http://check.my.awesome.ingress.com/$1' \
  --class="traefik" \
  --annotation traefik.frontend.passHostHeader="true" \
  --annotation traefik.backend.loadbalancer.sticky="true" \
  --annotation traefik.ingress.kubernetes.io/rule-type=PathPrefixStrip \
  --dry-run=client -o yaml | k apply -f-

# if doesn't work try to check:
k describe ingress traefik-app | grep error

# obtain the ingress external ip
external_ip=$(k get ingress traefik-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

# test the loadbalancer external ip
curl -D - --header "Host: check.my.awesome.ingress.com" "http://$external_ip"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# create a deployment (i.e. game 2048)
k create deployment game2048-3 --image=alexwhen/docker-2048

# expose the deployments using a LoadBalancer
# --target-port='': Name or number for the port on the container that the service should direct traffic to. Optional.
k expose deployment game2048-3 --port=80 --target-port 80 --type=ClusterIP
# or
# k create service clusterip game2048-3 --tcp 80:80

# gen certs with minica or openssl
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./tls.key -out ./tls.crt -subj "/CN=*.my.awesome.ingress.com"
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./tls.key -out ./tls.crt -subj "/CN=check.my.awesome.ingress.com"

# create secret
kubectl create secret tls game2048-3 --key ./tls.key --cert ./tls.crt

# create ingress, note we will get 404 from containred nginx (game-2048) with aliases: some*, v1, v2 because them doesn't exist in our container.
k create ingress traefik-app-tls \
  --rule='check.my.awesome.ingress.com/=game2048-3:80,tls=game2048-3' \
  --class="traefik" \
  --annotation traefik.ingress.kubernetes.io/router.tls="true" \
  --annotation traefik.ingress.kubernetes.io/router.entrypoints="websecure" \
  --annotation traefik.frontend.passHostHeader="true" \
  --annotation traefik.backend.loadbalancer.sticky="true" \
  --dry-run=client -o yaml | k apply -f-

# obtain the ingress external ip
external_ip=$(k get ingress traefik-app-tls -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

# test the loadbalancer external ip
echo | openssl s_client -showcerts -servername check.my.awesome.ingress.com -connect check.my.awesome.ingress.com:443 2>/dev/null | openssl x509 -inform pem -noout -text
# curl -vI -D - --header "Host: check.my.awesome.ingress.com" -k "https://check.my.awesome.ingress.com/"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# create a deployment (i.e. game 2048)
k create deployment game2048-4 --image=alexwhen/docker-2048

# expose the deployments using a LoadBalancer
k create service nodeport game2048-4 --tcp=80:80 --node-port=30080 --dry-run=client -o yaml | k apply -f-

# obtain the node ip addresses
k get nodes -o go-template='{{range .items}}{{range $elem := .status.addresses}}{{if eq $elem.type "InternalIP"}}{{$elem.address}}{{end}}{{end}}{{"\n"}}{{end}}' | xargs echo
  172.18.0.5 172.18.0.4 172.18.0.2 172.18.0.3

declare -a external_ips=(172.18.0.5 172.18.0.4 172.18.0.2 172.18.0.3)

# test the loadbalancer external ip
for i in $external_ips[@]; do nmap -p 30080 "$i"; done
for i in $external_ips[@]; do curl -D - "$i:30080" -o /dev/null; done

1

k get nodes -o go-template='{{range .items}}{{.metadata.name}}{{" :) "}}{{.status.addresses}}{{"\n"}}{{end}}'

1
2
3
4
5
6
7
8
9

cat > /tmp/template.gotemplate
{{range .items}}{{.metadata.name}}{{" :) "}}{{.status.addresses}}{{"\n"}}{{end}}
k get nodes -o go-template-file=/tmp/template.gotemplate

cat > /tmp/template.gotemplate
{{range .items}}
  {{.metadata.name}}{{" - "}}{{range $elem := .status.addresses}}{{if eq $elem.type "InternalIP"}}{{$elem.address}}{{end}}{{end}}{{"\n"}}
{{end}}
k get nodes -o go-template-file=/tmp/template.gotemplate