nomad.sh (7803B)
1 #!/usr/bin/env bash 2 # 3 # Packer shell provisioner for HashiCorp Nomad on Raspberry Pi 4 # https://www.nomadproject.io/docs/install/production/deployment-guide 5 6 # set -o errexit 7 # set -o nounset 8 set -o xtrace 9 10 cd "/home/${USERNAME}" 11 12 # Install podman driver for Nomad 13 # https://developer.hashicorp.com/nomad/plugins/drivers/community/containerd 14 curl -LO "https://releases.hashicorp.com/nomad-driver-podman/${NOMAD_PODMAN_DRIVER_VERSION}/nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_${ARCHITECTURE}.zip" 15 unzip "nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_${ARCHITECTURE}.zip" 16 mkdir -p /opt/nomad/plugins/ 17 mv nomad-driver-podman /opt/nomad/plugins/ 18 19 # Configure overlay storage for Podman 20 cat <<EOF > /etc/containers/storage.conf 21 [storage] 22 driver="overlay" 23 runroot = "/var/run/containers/storage" 24 graphroot = "/var/lib/containers/storage" 25 EOF 26 27 # Create Nomad data directory 28 mkdir -p /opt/nomad 29 30 # Prepare Nomad config folders 31 mkdir -p /etc/nomad.d 32 rm -rf /etc/nomad.d/* 33 34 # Move uploaded tls files 35 TLS_DIR=/etc/nomad.d/tls 36 mkdir -p $TLS_DIR 37 mv /tmp/tls/* $TLS_DIR 38 39 chmod 640 $TLS_DIR/*.pem 40 chmod 644 $TLS_DIR/dc1-{cli,client,server}* 41 chmod 644 $TLS_DIR/nomad-agent-ca.pem 42 chmod 644 $TLS_DIR/nomad-agent-ca.p12 43 44 # Nomad config files 45 cat << EOF > /etc/nomad.d/nomad.hcl 46 datacenter = "dc1" 47 data_dir = "/opt/nomad" 48 49 vault { 50 # https://www.nomadproject.io/docs/configuration/vault 51 # Nomad servers require a Vault Token set in /etc/nomad.d/nomad.env 52 # Nomad clients use the use the allocation's Vault token 53 enabled = true 54 address = "$VAULT_ADDR" 55 tls_skip_verify = true 56 $(if [[ "$NOMAD_SERVER" = true ]]; then 57 # Only servers create further Tokens for clients from the Token role 58 # https://developer.hashicorp.com/nomad/docs/configuration/vault#nomad-client 59 echo -e ' 60 # default workload identity for bao 61 # https://developer.hashicorp.com/nomad/docs/configuration/vault#default_identity 62 default_identity { 63 aud = ["vault.in0rdr.ch"] 64 ttl = "1h" 65 }' 66 fi) 67 } 68 69 telemetry { 70 # https://developer.hashicorp.com/nomad/docs/configuration/telemetry 71 publish_allocation_metrics = false 72 publish_node_metrics = true 73 } 74 EOF 75 76 77 # Configure as a Nomad server agent 78 # 79 # https://www.nomadproject.io/docs/configuration 80 # Note that it is strongly recommended not to operate a node as both client and server, 81 # although this is supported to simplify development and testing. 82 if [[ "$NOMAD_SERVER" = true ]]; then 83 cat << EOF > /etc/nomad.d/server.hcl 84 server { 85 enabled = true 86 87 # Encrypt gossip communication 88 encrypt = "$NOMAD_ENCRYPT" 89 90 bootstrap_expect = 3 91 server_join { 92 retry_join = [ "pi0", "pi2", "pi4" ] 93 } 94 95 # https://developer.hashicorp.com/nomad/docs/configuration/server#raft_multiplier 96 # https://developer.hashicorp.com/consul/docs/install/performance 97 raft_multiplier = 5 98 # https://developer.hashicorp.com/nomad/docs/configuration/server#client-heartbeats 99 # Increase heartbeat ttl under unreliable network conditions to prevent 100 # client: error heartbeating. retrying: error="failed to update status: rpc 101 # error: Not ready to serve consistent reads" 102 heartbeat_grace = "30s" 103 min_heartbeat_ttl = "30s" 104 default_scheduler_config { 105 scheduler_algorithm = "spread" 106 memory_oversubscription_enabled = true 107 reject_job_registration = false 108 pause_eval_broker = false 109 110 preemption_config { 111 batch_scheduler_enabled = true 112 system_scheduler_enabled = true 113 service_scheduler_enabled = true 114 sysbatch_scheduler_enabled = true 115 } 116 } 117 } 118 119 tls { 120 http = true 121 rpc = true 122 123 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 124 cert_file = "$TLS_DIR/dc1-server-nomad.pem" 125 key_file = "$TLS_DIR/dc1-server-nomad-key.pem" 126 127 verify_server_hostname = true 128 verify_https_client = false 129 } 130 EOF 131 fi # endif NOMAD_SERVER 132 133 systemctl enable nomad 134 systemctl enable podman 135 136 if [[ "$NOMAD_CLIENT" = true ]]; then 137 # this instance acts as a Nomad client agent 138 cat << EOF > /etc/nomad.d/client.hcl 139 client { 140 enabled = true 141 server_join { 142 retry_join = [ "pi0", "pi2", "pi4" ] 143 } 144 145 # Allow jobs to use the loopback interface for exposing 146 # private services only accessible from the client nodes 147 host_network "private" { 148 cidr = "127.0.0.1/32" 149 } 150 151 # Avoid resource contention between host/workload when oversubscribing 152 # memory. Reserve a portion of the node's resources from receiving tasks. 153 # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription 154 reserved { 155 # CPU to reserve, in MHz 156 cpu = 500 157 # memory to reserve, in MB 158 memory = 256 159 } 160 } 161 162 tls { 163 http = true 164 rpc = true 165 166 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 167 cert_file = "$TLS_DIR/dc1-client-nomad.pem" 168 key_file = "$TLS_DIR/dc1-client-nomad-key.pem" 169 170 verify_server_hostname = true 171 verify_https_client = false 172 } 173 174 # Enable raw exec driver (jobs w/o isolation) 175 plugin "raw_exec" { 176 config { 177 enabled = true 178 } 179 } 180 181 # Enable Podman task driver plugin 182 # Since Nomad 1.10.0 plugin binaries not loaded automatically 183 # https://developer.hashicorp.com/nomad/docs/upgrade/upgrade-specific#loading-binaries-from-plugin_dir-without-configuration 184 plugin "nomad-driver-podman" { 185 } 186 EOF 187 188 # Mount NFS for hostpath volumes 189 if [[ -n "$NFS_SERVER" ]]; then 190 mkdir -p "$NFS_MOUNT_TARGET" 191 # mount $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET 192 cat << EOF >> /etc/fstab 193 $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET nfs defaults 0 0 194 EOF 195 fi # endif NFS_SERVER 196 197 # Enable memory cgroups for NFS CSI 198 # https://ubuntu.com/tutorials/how-to-kubernetes-cluster-on-raspberry-pi#4-installing-microk8s 199 # https://downey.io/blog/exploring-cgroups-raspberry-pi 200 sed -i 's/$/ cgroup_enable=memory/' /boot/cmdline.txt 201 202 # Install CNI plugins 203 # - https://developer.hashicorp.com/nomad/docs/networking/cni 204 curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-${ARCHITECTURE}-v1.6.2.tgz" 205 mkdir -p /opt/cni/bin 206 tar -C /opt/cni/bin -xzf cni-plugins.tgz 207 208 # Prepare a Jenkins user for Jenkins agents and workload 209 # https://code.in0rdr.ch/nomad/file/docker/docker-jenkins-inbound-agent/README.html 210 /usr/sbin/groupadd -g $NOMAD_JENKINS_GID jenkins 211 /usr/sbin/useradd -m -s /bin/bash -u $NOMAD_JENKINS_UID -g $NOMAD_JENKINS_GID jenkins 212 213 # Keep my podman socket enabled even if no jenkins user is logged in. "loginctl 214 # enable-linger jenkins" command does not work when systemd is not booted with 215 # init system. Configuring KillUserProcesses directly in the user service as 216 # alternative. 217 # https://www.freedesktop.org/software/systemd/man/latest/logind.conf.html#KillUserProcesses= 218 mkdir -p /home/jenkins/.config/systemd/user/sockets.target.wants 219 cat << EOF >> /home/jenkins/.config/systemd/user/sockets.target.wants/podman.socket 220 [Unit] 221 Description=Podman API Socket 222 Documentation=man:podman-system-service(1) 223 224 [Socket] 225 ListenStream=%t/podman/podman.sock 226 SocketMode=0660 227 228 [Install] 229 WantedBy=sockets.target 230 231 [Login] 232 KillUserProcesses=no 233 EOF 234 235 # keep my podman.socket enabled even if no jenkins user is logged in 236 #loginctl enable-linger jenkins 237 # https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#-M 238 #systemctl --user -M jenkins@ start podman.socket 239 240 # create the mountpoint for the workspaces, podman does not create it for us 241 mkdir /home/jenkins/workspace 242 243 # move Nomad server truststore 244 mv $TLS_DIR/nomad-agent-ca.p12 /home/jenkins/ 245 246 # set ownership 247 chown -R jenkins: /home/jenkins 248 249 fi # endif NOMAD_CLIENT 250 251 # Configure .bashrc 252 cat << EOF >> "/home/${USERNAME}/.bashrc" 253 254 complete -C /usr/bin/nomad nomad 255 export NOMAD_ADDR=https://127.0.0.1:4646 256 export NOMAD_CACERT=$TLS_DIR/nomad-agent-ca.pem 257 export NOMAD_CLIENT_CERT=$TLS_DIR/dc1-cli-nomad.pem 258 export NOMAD_CLIENT_KEY=$TLS_DIR/dc1-cli-nomad-key.pem 259 EOF