nomad.sh (7829B)
1 #!/usr/bin/env bash 2 # 3 # Packer shell provisioner for HashiCorp Nomad on Raspberry Pi 4 # https://www.nomadproject.io/docs/install/production/deployment-guide 5 6 # set -o errexit 7 # set -o nounset 8 set -o xtrace 9 10 cd "/home/${USERNAME}" 11 12 # Install podman driver for Nomad 13 # https://developer.hashicorp.com/nomad/plugins/drivers/community/containerd 14 curl -LO "https://releases.hashicorp.com/nomad-driver-podman/${NOMAD_PODMAN_DRIVER_VERSION}/nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_${ARCHITECTURE}.zip" 15 unzip "nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_${ARCHITECTURE}.zip" 16 mkdir -p /opt/nomad/plugins/ 17 mv nomad-driver-podman /opt/nomad/plugins/ 18 19 # Configure overlay storage for Podman 20 cat <<EOF > /etc/containers/storage.conf 21 [storage] 22 driver="overlay" 23 runroot = "/var/run/containers/storage" 24 graphroot = "/var/lib/containers/storage" 25 EOF 26 27 # Create Nomad data directory 28 mkdir -p /opt/nomad 29 30 # Prepare Nomad config folders 31 mkdir -p /etc/nomad.d 32 rm -rf /etc/nomad.d/* 33 34 # Move uploaded tls files 35 TLS_DIR=/etc/nomad.d/tls 36 mkdir -p $TLS_DIR 37 mv /tmp/tls/* $TLS_DIR 38 39 chmod 640 $TLS_DIR/*.pem 40 chmod 644 $TLS_DIR/dc1-{cli,client,server}* 41 chmod 644 $TLS_DIR/nomad-agent-ca.pem 42 chmod 644 $TLS_DIR/nomad-agent-ca.p12 43 44 # Nomad config files 45 cat << EOF > /etc/nomad.d/nomad.hcl 46 datacenter = "dc1" 47 data_dir = "/opt/nomad" 48 49 vault { 50 # https://www.nomadproject.io/docs/configuration/vault 51 # Nomad servers require a Vault Token set in /etc/nomad.d/nomad.env 52 # Nomad clients use the use the allocation's Vault token 53 enabled = true 54 address = "$VAULT_ADDR" 55 tls_skip_verify = true 56 $(if [[ "$NOMAD_SERVER" = true ]]; then 57 # Only servers create further Tokens for clients from the Token role 58 # https://developer.hashicorp.com/nomad/docs/configuration/vault#nomad-client 59 echo -e ' 60 # default workload identity for bao 61 # https://developer.hashicorp.com/nomad/docs/configuration/vault#default_identity 62 default_identity { 63 aud = ["vault.in0rdr.ch"] 64 ttl = "1h" 65 }' 66 fi) 67 } 68 69 telemetry { 70 # https://developer.hashicorp.com/nomad/docs/configuration/telemetry 71 publish_allocation_metrics = false 72 publish_node_metrics = true 73 } 74 75 acl { 76 enabled = true 77 } 78 EOF 79 80 81 # Configure as a Nomad server agent 82 # 83 # https://www.nomadproject.io/docs/configuration 84 # Note that it is strongly recommended not to operate a node as both client and server, 85 # although this is supported to simplify development and testing. 86 if [[ "$NOMAD_SERVER" = true ]]; then 87 cat << EOF > /etc/nomad.d/server.hcl 88 server { 89 enabled = true 90 91 # Encrypt gossip communication 92 encrypt = "$NOMAD_ENCRYPT" 93 94 bootstrap_expect = 3 95 server_join { 96 retry_join = [ "pi0", "pi2", "pi4" ] 97 } 98 99 # https://developer.hashicorp.com/nomad/docs/configuration/server#raft_multiplier 100 # https://developer.hashicorp.com/consul/docs/install/performance 101 raft_multiplier = 5 102 # https://developer.hashicorp.com/nomad/docs/configuration/server#client-heartbeats 103 # Increase heartbeat ttl under unreliable network conditions to prevent 104 # client: error heartbeating. retrying: error="failed to update status: rpc 105 # error: Not ready to serve consistent reads" 106 heartbeat_grace = "30s" 107 min_heartbeat_ttl = "30s" 108 default_scheduler_config { 109 scheduler_algorithm = "spread" 110 memory_oversubscription_enabled = true 111 reject_job_registration = false 112 pause_eval_broker = false 113 114 preemption_config { 115 batch_scheduler_enabled = true 116 system_scheduler_enabled = true 117 service_scheduler_enabled = true 118 sysbatch_scheduler_enabled = true 119 } 120 } 121 } 122 123 tls { 124 http = true 125 rpc = true 126 127 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 128 cert_file = "$TLS_DIR/dc1-server-nomad.pem" 129 key_file = "$TLS_DIR/dc1-server-nomad-key.pem" 130 131 verify_server_hostname = true 132 verify_https_client = false 133 } 134 EOF 135 fi # endif NOMAD_SERVER 136 137 systemctl enable nomad 138 systemctl enable podman 139 140 if [[ "$NOMAD_CLIENT" = true ]]; then 141 # this instance acts as a Nomad client agent 142 cat << EOF > /etc/nomad.d/client.hcl 143 client { 144 enabled = true 145 server_join { 146 retry_join = [ "pi0", "pi2", "pi4" ] 147 } 148 149 # Allow jobs to use the loopback interface for exposing 150 # private services only accessible from the client nodes 151 host_network "private" { 152 cidr = "127.0.0.1/32" 153 } 154 155 # Avoid resource contention between host/workload when oversubscribing 156 # memory. Reserve a portion of the node's resources from receiving tasks. 157 # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription 158 reserved { 159 # CPU to reserve, in MHz 160 cpu = 500 161 # memory to reserve, in MB 162 memory = 256 163 } 164 } 165 166 tls { 167 http = true 168 rpc = true 169 170 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 171 cert_file = "$TLS_DIR/dc1-client-nomad.pem" 172 key_file = "$TLS_DIR/dc1-client-nomad-key.pem" 173 174 verify_server_hostname = true 175 verify_https_client = false 176 } 177 178 # Enable raw exec driver (jobs w/o isolation) 179 plugin "raw_exec" { 180 config { 181 enabled = true 182 } 183 } 184 185 # Enable Podman task driver plugin 186 # Since Nomad 1.10.0 plugin binaries not loaded automatically 187 # https://developer.hashicorp.com/nomad/docs/upgrade/upgrade-specific#loading-binaries-from-plugin_dir-without-configuration 188 plugin "nomad-driver-podman" { 189 } 190 EOF 191 192 # Mount NFS for hostpath volumes 193 if [[ -n "$NFS_SERVER" ]]; then 194 mkdir -p "$NFS_MOUNT_TARGET" 195 # mount $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET 196 cat << EOF >> /etc/fstab 197 $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET nfs defaults 0 0 198 EOF 199 fi # endif NFS_SERVER 200 201 # Enable memory cgroups for NFS CSI 202 # https://ubuntu.com/tutorials/how-to-kubernetes-cluster-on-raspberry-pi#4-installing-microk8s 203 # https://downey.io/blog/exploring-cgroups-raspberry-pi 204 sed -i 's/$/ cgroup_enable=memory/' /boot/cmdline.txt 205 206 # Install CNI plugins 207 # - https://developer.hashicorp.com/nomad/docs/networking/cni 208 curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-${ARCHITECTURE}-v1.6.2.tgz" 209 mkdir -p /opt/cni/bin 210 tar -C /opt/cni/bin -xzf cni-plugins.tgz 211 212 # Prepare a Jenkins user for Jenkins agents and workload 213 # https://code.in0rdr.ch/nomad/file/docker/docker-jenkins-inbound-agent/README.html 214 /usr/sbin/groupadd -g $NOMAD_JENKINS_GID jenkins 215 /usr/sbin/useradd -m -s /bin/bash -u $NOMAD_JENKINS_UID -g $NOMAD_JENKINS_GID jenkins 216 217 # Keep my podman socket enabled even if no jenkins user is logged in. "loginctl 218 # enable-linger jenkins" command does not work when systemd is not booted with 219 # init system. Configuring KillUserProcesses directly in the user service as 220 # alternative. 221 # https://www.freedesktop.org/software/systemd/man/latest/logind.conf.html#KillUserProcesses= 222 mkdir -p /home/jenkins/.config/systemd/user/sockets.target.wants 223 cat << EOF >> /home/jenkins/.config/systemd/user/sockets.target.wants/podman.socket 224 [Unit] 225 Description=Podman API Socket 226 Documentation=man:podman-system-service(1) 227 228 [Socket] 229 ListenStream=%t/podman/podman.sock 230 SocketMode=0660 231 232 [Install] 233 WantedBy=sockets.target 234 235 [Login] 236 KillUserProcesses=no 237 EOF 238 239 # keep my podman.socket enabled even if no jenkins user is logged in 240 #loginctl enable-linger jenkins 241 # https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#-M 242 #systemctl --user -M jenkins@ start podman.socket 243 244 # create the mountpoint for the workspaces, podman does not create it for us 245 mkdir /home/jenkins/workspace 246 247 # move Nomad server truststore 248 mv $TLS_DIR/nomad-agent-ca.p12 /home/jenkins/ 249 250 # set ownership 251 chown -R jenkins: /home/jenkins 252 253 fi # endif NOMAD_CLIENT 254 255 # Configure .bashrc 256 cat << EOF >> "/home/${USERNAME}/.bashrc" 257 258 complete -C /usr/bin/nomad nomad 259 export NOMAD_ADDR=https://127.0.0.1:4646 260 export NOMAD_CACERT=$TLS_DIR/nomad-agent-ca.pem 261 export NOMAD_CLIENT_CERT=$TLS_DIR/dc1-cli-nomad.pem 262 export NOMAD_CLIENT_KEY=$TLS_DIR/dc1-cli-nomad-key.pem 263 EOF