commit 01db098e53e63f33b42c4d6dcbbf5dbc6cc4c551
parent 2352328373c6c7526519e121070f841b83341041
Author: Andreas Gruhler <andreas.gruhler@adfinis.com>
Date: Mon, 15 May 2023 11:36:38 +0200
feat(nomad): add cilium CNI
Diffstat:
| M | bootstrap.sh | | | 4 | ++-- |
| M | nomad.sh | | | 109 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ |
2 files changed, 104 insertions(+), 9 deletions(-)
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -13,8 +13,8 @@ echo "${HOSTNAME}" > /etc/hostname
cat << EOF > /etc/hosts
127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
-ff02::1 ip6-allnodes
-ff02::2 ip6-allrouters
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
127.0.1.1 $HOSTNAME
EOF
diff --git a/nomad.sh b/nomad.sh
@@ -64,13 +64,19 @@ client {
}
}
-# CSI Node plugins must run as privileged Docker jobs
-# because they use bidirectional mount propagation
-# in order to mount disks to the underlying host:
-# https://learn.hashicorp.com/tutorials/nomad/stateful-workloads-csi-volumes
plugin "docker" {
config {
+ # CSI Node plugins must run as privileged Docker jobs
+ # because they use bidirectional mount propagation
+ # in order to mount disks to the underlying host:
+ # https://learn.hashicorp.com/tutorials/nomad/stateful-workloads-csi-volumes
allow_privileged = true
+ volumes {
+ # Netreap Cilium operator connects to Cilium agent on the node through Unix
+ # socket on shared host path volume
+ # https://cosmonic.com/blog/engineering/netreap-a-practical-guide-to-running-cilium-in-nomad
+ enabled = true
+ }
}
}
@@ -100,7 +106,6 @@ vault {
tls_skip_verify = true
create_from_role = "nomad-cluster"
}
-
EOF
# Configure systemd service unit
@@ -154,8 +159,98 @@ fi
# https://downey.io/blog/exploring-cgroups-raspberry-pi
sed -i 's/$/ cgroup_enable=memory/' /boot/cmdline.txt
-# Install CNI plugin for Consul connect integration
-# https://www.nomadproject.io/docs/integrations/consul-connect
+# Ensure iptable modules for Netreap
+cat << EOF > /etc/modules-load.d/iptables.conf
+iptable_nat
+iptable_mangle
+iptable_raw
+iptable_filter
+ip6table_mangle
+ip6table_raw
+ip6table_filter
+EOF
+
+# prepare Consul TLS config for Cilium
+mkdir /etc/cilium
+cat << EOF > /etc/cilium/consul-tlsconfig.yaml
+---
+# https://docs.cilium.io/en/v1.13/cmdref/kvstore
+# https://github.com/cilium/cilium/blob/main/pkg/kvstore/consul.go
+# https://github.com/hashicorp/consul/blob/main/api/api.go
+# https://github.com/cilium/cilium/pull/6260
+cafile: /var/lib/cilium/consul-tls/consul-agent-ca.pem
+certfile: /var/lib/cilium/consul-tls/dc1-server-consul.pem
+keyfile: /var/lib/cilium/consul-tls/dc1-server-consul-key.pem
+#insecureSkipVerify: true
+EOF
+
+# Run Cilium as privileged container on the node
+cat << 'EOF' > /etc/systemd/system/cilium.service
+[Unit]
+Description=Cilium Agent
+After=docker.service
+Requires=docker.service
+After=consul.service
+Wants=consul.service
+Before=nomad.service
+
+[Service]
+Restart=always
+ExecStartPre=-/usr/bin/docker exec %n stop
+ExecStartPre=-/usr/bin/docker rm %n
+
+ExecStart=/usr/bin/docker run --rm --name %n \
+ -v /var/run/cilium:/var/run/cilium \
+ -v /sys/fs/bpf:/sys/fs/bpf \
+ -v /opt/consul/tls/:/var/lib/cilium/consul-tls \
+ -v /etc/cilium/consul-tlsconfig.yaml:/var/lib/cilium/consul-tlsconfig.yaml \
+ --net=host \
+ --cap-add NET_ADMIN \
+ --cap-add NET_RAW \
+ --cap-add IPC_LOCK \
+ --cap-add SYS_MODULE \
+ --cap-add SYS_ADMIN \
+ --cap-add SYS_RESOURCE \
+ --privileged \
+ cilium/cilium:v1.13.2 \
+ cilium-agent --kvstore consul \
+ --kvstore-opt consul.address=https://127.0.0.1:8501,consul.tlsconfig=/var/lib/cilium/consul-tlsconfig.yaml \
+ --enable-ipv6=false \
+ --enable-l7-proxy=false \
+ --ipv4-range 172.16.0.0/16
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl enable cilium
+
+# Configure Cilium CNI
+mkdir /opt/cni/conf
+cat << EOF > /opt/cni/conf/cilium.conflist
+{
+ "name": "cilium",
+ "cniVersion": "1.0.0",
+ "plugins": [
+ {
+ "type": "cilium-cni",
+ "enable-debug": false
+ }
+ ]
+}
+EOF
+
+# Install Cilium CNI and binaries to node
+docker run --rm --entrypoint bash -v /tmp:/out cilium/cilium:v1.13.2 \
+ -c 'cp /usr/bin/cilium* /out; cp /opt/cni/bin/cilium-cni /out'
+mv /tmp/cilium-cni /opt/cni/bin/cilium-cni
+mv /tmp/cilium* /usr/local/bin
+
+# Install CNI plugin for Consul connect integration for Consul snapshot tasks
+# and Netreap Cilium operator
+# - https://www.nomadproject.io/docs/integrations/consul-connect
+# - https://github.com/cosmonic/netreap
+# - https://cosmonic.com/blog/engineering/netreap-a-practical-guide-to-running-cilium-in-nomad
curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-arm64-v1.1.1.tgz"
mkdir -p /opt/cni/bin
tar -C /opt/cni/bin -xzf cni-plugins.tgz
