commit c65c4c90757f3fda597ba73c67016ee63131116a
parent 1bfb90b40af14d46e5e9e4f92fdefb4c808b247d
Author: Andreas Gruhler <andreas.gruhler@adfinis.com>
Date: Thu, 18 May 2023 19:40:22 +0200
feat: podman driver
* feat: upgrade consul, vault, raspios
* feat(cilium): extract binaries w/ cloud-init
* fix: missing directories
* feat(podman): add insecure local registry
* feat: replace docker with podman
Diffstat:
3 files changed, 83 insertions(+), 44 deletions(-)
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -26,8 +26,17 @@ mv /tmp/resizerootfs /usr/sbin/
systemctl enable resizerootfs.service
# Install packages
-curl -sSL https://get.docker.com | sh
-DEBIAN_FRONTEND=noninteractive apt-get install -y jq
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y jq podman cloud-init
+
+# Configure insecure local registry
+cat << EOF > /etc/containers/registries.conf
+unqualified-search-registries = ['127.0.0.1:5000', 'docker.io']
+
+[[registry]]
+location = "127.0.0.1:5000"
+insecure = true
+EOF
# Set up no-password sudo
rm /etc/sudoers.d/010_pi-nopasswd
diff --git a/hashi-pi.json b/hashi-pi.json
@@ -3,11 +3,11 @@
"hostname": "HashiPi0",
"username": "in0rdr",
"authorized_keys": "",
- "img_url": "https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2022-09-26/2022-09-22-raspios-bullseye-arm64-lite.img.xz",
+ "img_url": "https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2023-05-03/2023-05-03-raspios-bullseye-arm64-lite.img.xz",
"img_name": "raspi.img",
"flash_device_path": null,
"img_size": "3G",
- "consul_version": "1.14.4",
+ "consul_version": "1.15.2",
"consul_arch": "arm64",
"consul_encrypt": null,
"consul_tls_ca": null,
@@ -18,7 +18,7 @@
"nomad_nfs_server": "",
"nomad_nfs_mount": "",
"nomad_nfs_target": "",
- "vault_version": "1.12.3",
+ "vault_version": "1.13.2",
"vault_arch": "arm64",
"vault_tls_ca_cert": "./tls/vault/ca/vault_ca.pem",
"vault_tls_ca_key": "./tls/vault/ca/vault_ca.key",
diff --git a/nomad.sh b/nomad.sh
@@ -11,6 +11,14 @@ NOMAD_URL="https://releases.hashicorp.com/nomad"
cd "/home/${USERNAME}"
+# Install podman driver for Nomad
+# https://developer.hashicorp.com/nomad/plugins/drivers/community/containerd
+NOMAD_PODMAN_DRIVER_VERSION=0.4.2
+curl -LO "https://releases.hashicorp.com/nomad-driver-podman/${NOMAD_PODMAN_DRIVER_VERSION}/nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip"
+unzip "nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip"
+mkdir -p /opt/nomad/plugins/
+mv nomad-driver-podman /opt/nomad/plugins/
+
# Download Nomad binary and checksums
curl -sS -O "${NOMAD_URL}/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_${NOMAD_ARCH}.zip"
curl -sS -O "${NOMAD_URL}/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_SHA256SUMS"
@@ -64,19 +72,15 @@ client {
}
}
-plugin "docker" {
+plugin "nomad-driver-podman" {
config {
- # CSI Node plugins must run as privileged Docker jobs
- # because they use bidirectional mount propagation
- # in order to mount disks to the underlying host:
- # https://learn.hashicorp.com/tutorials/nomad/stateful-workloads-csi-volumes
- allow_privileged = true
volumes {
- # Netreap Cilium operator connects to Cilium agent on the node through Unix
- # socket on shared host path volume
+ # Netreap Cilium operator connects to Cilium agent on the node through
+ # Unix socket on shared host path volume
# https://cosmonic.com/blog/engineering/netreap-a-practical-guide-to-running-cilium-in-nomad
enabled = true
}
+ recover_stopped = false
}
}
@@ -171,7 +175,7 @@ ip6table_filter
EOF
# prepare Consul TLS config for Cilium
-mkdir /etc/cilium
+mkdir -p /etc/cilium
cat << EOF > /etc/cilium/consul-tlsconfig.yaml
---
# https://docs.cilium.io/en/v1.13/cmdref/kvstore
@@ -185,48 +189,43 @@ keyfile: /var/lib/cilium/consul-tls/dc1-server-consul-key.pem
EOF
# Run Cilium as privileged container on the node
-cat << 'EOF' > /etc/systemd/system/cilium.service
+CILIUM_VERSION=1.13.2
+cat << EOF > /etc/systemd/system/cilium.service
[Unit]
Description=Cilium Agent
-After=docker.service
-Requires=docker.service
After=consul.service
Wants=consul.service
Before=nomad.service
[Service]
Restart=always
-ExecStartPre=-/usr/bin/docker exec %n stop
-ExecStartPre=-/usr/bin/docker rm %n
-
-ExecStart=/usr/bin/docker run --rm --name %n \
- -v /var/run/cilium:/var/run/cilium \
- -v /sys/fs/bpf:/sys/fs/bpf \
- -v /opt/consul/tls/:/var/lib/cilium/consul-tls \
- -v /etc/cilium/consul-tlsconfig.yaml:/var/lib/cilium/consul-tlsconfig.yaml \
- --net=host \
- --cap-add NET_ADMIN \
- --cap-add NET_RAW \
- --cap-add IPC_LOCK \
- --cap-add SYS_MODULE \
- --cap-add SYS_ADMIN \
- --cap-add SYS_RESOURCE \
- --privileged \
- cilium/cilium:v1.13.2 \
- cilium-agent --kvstore consul \
- --kvstore-opt consul.address=https://127.0.0.1:8501,consul.tlsconfig=/var/lib/cilium/consul-tlsconfig.yaml \
- --enable-ipv6=false \
- --enable-l7-proxy=false \
+ExecStartPre=-/usr/bin/podman exec %n stop
+ExecStartPre=-/usr/bin/podman rm %n
+ExecStartPre=-/usr/bin/mkdir -p /var/run/cilium
+
+ExecStart=/usr/bin/podman run --rm --name %n \\
+ -v /var/run/cilium:/var/run/cilium \\
+ -v /sys/fs/bpf:/sys/fs/bpf \\
+ -v /opt/consul/tls/:/var/lib/cilium/consul-tls \\
+ -v /etc/cilium/consul-tlsconfig.yaml:/var/lib/cilium/consul-tlsconfig.yaml \\
+ --net=host \\
+ --privileged \\
+ docker://cilium/cilium:v$CILIUM_VERSION \\
+ cilium-agent --kvstore consul \\
+ --kvstore-opt consul.address=https://127.0.0.1:8501,consul.tlsconfig=/var/lib/cilium/consul-tlsconfig.yaml \\
+ --enable-ipv6=false \\
+ --enable-l7-proxy=false \\
--ipv4-range 172.16.0.0/16
[Install]
WantedBy=multi-user.target
EOF
+mkdir -p /var/run/cilium
systemctl enable cilium
# Configure Cilium CNI
-mkdir /opt/cni/conf
+mkdir -p /opt/cni/conf
cat << EOF > /opt/cni/conf/cilium.conflist
{
"name": "cilium",
@@ -240,11 +239,42 @@ cat << EOF > /opt/cni/conf/cilium.conflist
}
EOF
-# Install Cilium CNI and binaries to node
-docker run --rm --entrypoint bash -v /tmp:/out cilium/cilium:v1.13.2 \
- -c 'cp /usr/bin/cilium* /out; cp /opt/cni/bin/cilium-cni /out'
-mv /tmp/cilium-cni /opt/cni/bin/cilium-cni
-mv /tmp/cilium* /usr/local/bin
+# Install Cilium CNI and binaries to node during first boot
+rm -rf /etc/cloud/*
+mkdir -p /etc/cloud/cloud.cfg.d
+
+cat << EOF > /etc/cloud/cloud.cfg
+# https://cloudinit.readthedocs.io/en/latest/explanation/boot.html
+# init modules are run before config modules
+cloud_init_modules:
+# create a script which will be run later by scripts-user
+# https://cloudinit.readthedocs.io/en/latest/reference/modules.html#runcmd
+- runcmd
+
+cloud_config_modules:
+# execute the runcmd scripts
+- scripts-user
+EOF
+
+cat << EOF > /etc/cloud/cloud.cfg.d/99_cilium.cfg
+#cloud-config
+# create a script which runs only on first boot
+runcmd:
+# create a cilium container, but does not start it yet
+- cid=\$(podman create "docker://cilium/cilium:v${CILIUM_VERSION}")
+# extract the cilium binaries from the container
+- podman cp "\$cid:/opt/cni/bin/cilium-cni" /opt/cni/bin/cilium-cni
+# unfortunately, docker cp does not support globbing yet
+- podman cp "\$cid:/usr/bin/cilium" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-agent" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-bugtool" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-envoy" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-health" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-health-responder" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-mount" /usr/local/bin/
+- podman cp "\$cid:/usr/bin/cilium-sysctlfix" /usr/local/bin/
+- podman rm "\$cid"
+EOF
# Install CNI plugin for Consul connect integration for Consul snapshot tasks
# and Netreap Cilium operator
