nomad.sh (6741B)
1 #!/usr/bin/env bash 2 # 3 # Packer shell provisioner for HashiCorp Nomad on Raspberry Pi 4 # https://www.nomadproject.io/docs/install/production/deployment-guide 5 6 # set -o errexit 7 # set -o nounset 8 set -o xtrace 9 10 cd "/home/${USERNAME}" 11 12 # Install podman driver for Nomad 13 # https://developer.hashicorp.com/nomad/plugins/drivers/community/containerd 14 curl -LO "https://releases.hashicorp.com/nomad-driver-podman/${NOMAD_PODMAN_DRIVER_VERSION}/nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip" 15 unzip "nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip" 16 mkdir -p /opt/nomad/plugins/ 17 mv nomad-driver-podman /opt/nomad/plugins/ 18 19 # Create Nomad data directory 20 mkdir -p /opt/nomad 21 22 # Prepare Nomad config folders 23 mkdir -p /etc/nomad.d 24 rm -rf /etc/nomad.d/* 25 26 # Move uploaded tls files 27 TLS_DIR=/etc/nomad.d/tls 28 mkdir -p $TLS_DIR 29 mv /tmp/tls/* $TLS_DIR 30 31 chmod 640 $TLS_DIR/*.pem 32 chmod 644 $TLS_DIR/dc1-{cli,client,server}* 33 chmod 644 $TLS_DIR/nomad-agent-ca.pem 34 chmod 644 $TLS_DIR/nomad-agent-ca.p12 35 36 # Nomad config files 37 cat << EOF > /etc/nomad.d/nomad.hcl 38 datacenter = "dc1" 39 data_dir = "/opt/nomad" 40 41 vault { 42 # https://www.nomadproject.io/docs/configuration/vault 43 # Nomad servers require a Vault Token set in /etc/nomad.d/nomad.env 44 # Nomad clients use the use the allocation's Vault token 45 enabled = true 46 address = "$VAULT_ADDR" 47 tls_skip_verify = true 48 $(if [[ "$NOMAD_SERVER" = true ]]; then 49 # Only servers create further Tokens for clients from the Token role 50 # https://developer.hashicorp.com/nomad/docs/configuration/vault#nomad-client 51 echo -e ' 52 # default workload identity for bao 53 # https://developer.hashicorp.com/nomad/docs/configuration/vault#default_identity 54 default_identity { 55 aud = ["vault.in0rdr.ch"] 56 ttl = "1h" 57 }' 58 fi) 59 } 60 61 telemetry { 62 # https://developer.hashicorp.com/nomad/docs/configuration/telemetry 63 publish_allocation_metrics = false 64 publish_node_metrics = true 65 } 66 EOF 67 68 69 # Configure as a Nomad server agent 70 # 71 # https://www.nomadproject.io/docs/configuration 72 # Note that it is strongly recommended not to operate a node as both client and server, 73 # although this is supported to simplify development and testing. 74 if [[ "$NOMAD_SERVER" = true ]]; then 75 cat << EOF > /etc/nomad.d/server.hcl 76 server { 77 enabled = true 78 79 # Encrypt gossip communication 80 encrypt = "$NOMAD_ENCRYPT" 81 82 bootstrap_expect = 3 83 server_join { 84 retry_join = [ "pi0", "pi2", "pi4" ] 85 } 86 87 # https://developer.hashicorp.com/nomad/docs/configuration/server#raft_multiplier 88 # https://developer.hashicorp.com/consul/docs/install/performance 89 raft_multiplier = 5 90 # https://developer.hashicorp.com/nomad/docs/configuration/server#client-heartbeats 91 # Increase heartbeat ttl under unreliable network conditions to prevent 92 # client: error heartbeating. retrying: error="failed to update status: rpc 93 # error: Not ready to serve consistent reads" 94 heartbeat_grace = "30s" 95 min_heartbeat_ttl = "30s" 96 default_scheduler_config { 97 scheduler_algorithm = "spread" 98 memory_oversubscription_enabled = true 99 reject_job_registration = false 100 pause_eval_broker = false 101 102 preemption_config { 103 batch_scheduler_enabled = true 104 system_scheduler_enabled = true 105 service_scheduler_enabled = true 106 sysbatch_scheduler_enabled = true 107 } 108 } 109 } 110 111 tls { 112 http = true 113 rpc = true 114 115 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 116 cert_file = "$TLS_DIR/dc1-server-nomad.pem" 117 key_file = "$TLS_DIR/dc1-server-nomad-key.pem" 118 119 verify_server_hostname = true 120 verify_https_client = false 121 } 122 EOF 123 fi # endif NOMAD_SERVER 124 125 systemctl enable nomad 126 127 if [[ "$NOMAD_CLIENT" = true ]]; then 128 # this instance acts as a Nomad client agent 129 cat << EOF > /etc/nomad.d/client.hcl 130 client { 131 enabled = true 132 server_join { 133 retry_join = [ "pi0", "pi2", "pi4" ] 134 } 135 136 # Allow jobs to use the loopback interface for exposing 137 # private services only accessible from the client nodes 138 host_network "private" { 139 cidr = "127.0.0.1/32" 140 } 141 142 # Avoid resource contention between host/workload when oversubscribing 143 # memory. Reserve a portion of the node's resources from receiving tasks. 144 # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription 145 reserved { 146 # CPU to reserve, in MHz 147 cpu = 500 148 # memory to reserve, in MB 149 memory = 256 150 } 151 } 152 153 tls { 154 http = true 155 rpc = true 156 157 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 158 cert_file = "$TLS_DIR/dc1-client-nomad.pem" 159 key_file = "$TLS_DIR/dc1-client-nomad-key.pem" 160 161 verify_server_hostname = true 162 verify_https_client = false 163 } 164 165 # Enable raw exec driver (jobs w/o isolation) 166 plugin "raw_exec" { 167 config { 168 enabled = true 169 } 170 } 171 EOF 172 173 # Mount NFS for hostpath volumes 174 if [[ -n "$NFS_SERVER" ]]; then 175 mkdir -p "$NFS_MOUNT_TARGET" 176 # mount $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET 177 cat << EOF >> /etc/fstab 178 $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET nfs defaults 0 0 179 EOF 180 fi # endif NFS_SERVER 181 182 # Enable memory cgroups for NFS CSI 183 # https://downey.io/blog/exploring-cgroups-raspberry-pi 184 sed -i 's/$/ cgroup_enable=memory/' /boot/cmdline.txt 185 186 # Ensure iptable modules for Netreap 187 cat << EOF > /etc/modules-load.d/iptables.conf 188 iptable_nat 189 iptable_mangle 190 iptable_raw 191 iptable_filter 192 ip6table_mangle 193 ip6table_raw 194 ip6table_filter 195 EOF 196 197 # Install CNI plugins 198 # - https://developer.hashicorp.com/nomad/docs/networking/cni 199 curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-arm64-v1.6.2.tgz" 200 mkdir -p /opt/cni/bin 201 tar -C /opt/cni/bin -xzf cni-plugins.tgz 202 203 # Prepare a Jenkins user for Jenkins agents and workload 204 # https://code.in0rdr.ch/nomad/file/docker/docker-jenkins-inbound-agent/README.html 205 groupadd -g $NOMAD_JENKINS_GID jenkins 206 useradd -m -s /bin/bash -u $NOMAD_JENKINS_UID -g $NOMAD_JENKINS_GID jenkins 207 208 # keep my podman.socket enabled even if no jenkins user is logged in 209 loginctl enable-linger jenkins 210 211 # https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#-M 212 systemctl --user -M jenkins@ start podman.socket 213 systemctl --user -M jenkins@ enable podman.socket 214 215 # create the mountpoint for the workspaces, podman does not create it for us 216 sudo -u jenkins mkdir /home/jenkins/workspace 217 218 # move Nomad server truststore 219 sudo mv $TLS_DIR/nomad-agent-ca.p12 /home/jenkins/ 220 sudo chown jenkins: /home/jenkins/nomad-agent-ca.p12 221 222 fi # endif NOMAD_CLIENT 223 224 # Configure .bashrc 225 cat << EOF >> "/home/${USERNAME}/.bashrc" 226 227 complete -C /usr/bin/nomad nomad 228 export NOMAD_ADDR=https://127.0.0.1:4646 229 export NOMAD_CACERT=$TLS_DIR/nomad-agent-ca.pem 230 export NOMAD_CLIENT_CERT=$TLS_DIR/dc1-cli-nomad.pem 231 export NOMAD_CLIENT_KEY=$TLS_DIR/dc1-cli-nomad-key.pem 232 EOF