nomad.sh (6729B)
1 #!/usr/bin/env bash 2 # 3 # Packer shell provisioner for HashiCorp Nomad on Raspberry Pi 4 # https://www.nomadproject.io/docs/install/production/deployment-guide 5 6 # set -o errexit 7 # set -o nounset 8 set -o xtrace 9 10 cd "/home/${USERNAME}" 11 12 # Move uploaded tls files 13 TLS_DIR=/etc/nomad.d/tls 14 mkdir -p $TLS_DIR 15 mv /tmp/tls/* $TLS_DIR 16 17 chmod 640 $TLS_DIR/*.pem 18 chmod 644 $TLS_DIR/dc1-{cli,client,server}* 19 chmod 644 $TLS_DIR/nomad-agent-ca.pem 20 chmod 644 $TLS_DIR/nomad-agent-ca.p12 21 22 # Install podman driver for Nomad 23 # https://developer.hashicorp.com/nomad/plugins/drivers/community/containerd 24 curl -LO "https://releases.hashicorp.com/nomad-driver-podman/${NOMAD_PODMAN_DRIVER_VERSION}/nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip" 25 unzip "nomad-driver-podman_${NOMAD_PODMAN_DRIVER_VERSION}_linux_arm64.zip" 26 mkdir -p /opt/nomad/plugins/ 27 mv nomad-driver-podman /opt/nomad/plugins/ 28 29 # Create Nomad data directory 30 mkdir -p /opt/nomad 31 32 # Create Nomads config files 33 mkdir -p /etc/nomad.d 34 rm -rf /etc/nomad.d/* 35 36 cat << EOF > /etc/nomad.d/nomad.hcl 37 datacenter = "dc1" 38 data_dir = "/opt/nomad" 39 40 vault { 41 # https://www.nomadproject.io/docs/configuration/vault 42 # Nomad servers require a Vault Token set in /etc/nomad.d/nomad.env 43 # Nomad clients use the use the allocation's Vault token 44 enabled = true 45 address = "https://127.0.0.1:8200" 46 tls_skip_verify = true 47 $(if [[ "$NOMAD_SERVER" = true ]]; then 48 # Only servers create further Tokens for clients from the Token role 49 # https://developer.hashicorp.com/nomad/docs/configuration/vault#nomad-client 50 echo -e " 51 # default workload identity for bao 52 # https://developer.hashicorp.com/nomad/docs/configuration/vault#default_identity 53 default_identity { 54 aud = ["vault.in0rdr.ch"] 55 ttl = "1h" 56 }" 57 fi) 58 } 59 60 telemetry { 61 # https://developer.hashicorp.com/nomad/docs/configuration/telemetry 62 publish_allocation_metrics = false 63 publish_node_metrics = true 64 } 65 EOF 66 67 68 # Configure as a Nomad server agent 69 # 70 # https://www.nomadproject.io/docs/configuration 71 # Note that it is strongly recommended not to operate a node as both client and server, 72 # although this is supported to simplify development and testing. 73 if [[ "$NOMAD_SERVER" = true ]]; then 74 cat << EOF > /etc/nomad.d/server.hcl 75 server { 76 enabled = true 77 78 # Encrypt gossip communication 79 encrypt = "$NOMAD_ENCRYPT" 80 81 bootstrap_expect = 3 82 server_join { 83 retry_join = [ "pi0", "pi2", "pi4" ] 84 } 85 86 # https://developer.hashicorp.com/nomad/docs/configuration/server#raft_multiplier 87 # https://developer.hashicorp.com/consul/docs/install/performance 88 raft_multiplier = 5 89 # https://developer.hashicorp.com/nomad/docs/configuration/server#client-heartbeats 90 # Increase heartbeat ttl under unreliable network conditions to prevent 91 # client: error heartbeating. retrying: error="failed to update status: rpc 92 # error: Not ready to serve consistent reads" 93 heartbeat_grace = "30s" 94 min_heartbeat_ttl = "30s" 95 default_scheduler_config { 96 scheduler_algorithm = "spread" 97 memory_oversubscription_enabled = true 98 reject_job_registration = false 99 pause_eval_broker = false 100 101 preemption_config { 102 batch_scheduler_enabled = true 103 system_scheduler_enabled = true 104 service_scheduler_enabled = true 105 sysbatch_scheduler_enabled = true 106 } 107 } 108 } 109 110 tls { 111 http = true 112 rpc = true 113 114 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 115 cert_file = "$TLS_DIR/dc1-server-nomad.pem" 116 key_file = "$TLS_DIR/dc1-server-nomad-key.pem" 117 118 verify_server_hostname = true 119 verify_https_client = false 120 } 121 EOF 122 fi # endif NOMAD_SERVER 123 124 systemctl enable nomad 125 126 if [[ "$NOMAD_CLIENT" = true ]]; then 127 # this instance acts as a Nomad client agent 128 cat << EOF > /etc/nomad.d/client.hcl 129 client { 130 enabled = true 131 server_join { 132 retry_join = [ "pi0", "pi2", "pi4" ] 133 } 134 135 # Allow jobs to use the loopback interface for exposing 136 # private services only accessible from the client nodes 137 host_network "private" { 138 cidr = "127.0.0.1/32" 139 } 140 141 # Avoid resource contention between host/workload when oversubscribing 142 # memory. Reserve a portion of the node's resources from receiving tasks. 143 # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription 144 reserved { 145 # CPU to reserve, in MHz 146 cpu = 500 147 # memory to reserve, in MB 148 memory = 256 149 } 150 } 151 152 tls { 153 http = true 154 rpc = true 155 156 ca_file = "$TLS_DIR/nomad-agent-ca.pem" 157 cert_file = "$TLS_DIR/dc1-client-nomad.pem" 158 key_file = "$TLS_DIR/dc1-client-nomad-key.pem" 159 160 verify_server_hostname = true 161 verify_https_client = false 162 } 163 164 # Enable raw exec driver (jobs w/o isolation) 165 plugin "raw_exec" { 166 config { 167 enabled = true 168 } 169 } 170 EOF 171 172 # Mount NFS for hostpath volumes 173 if [[ -n "$NFS_SERVER" ]]; then 174 mkdir -p "$NFS_MOUNT_TARGET" 175 # mount $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET 176 cat << EOF >> /etc/fstab 177 $NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET nfs defaults 0 0 178 EOF 179 fi # endif NFS_SERVER 180 181 # Enable memory cgroups for NFS CSI 182 # https://downey.io/blog/exploring-cgroups-raspberry-pi 183 sed -i 's/$/ cgroup_enable=memory/' /boot/cmdline.txt 184 185 # Ensure iptable modules for Netreap 186 cat << EOF > /etc/modules-load.d/iptables.conf 187 iptable_nat 188 iptable_mangle 189 iptable_raw 190 iptable_filter 191 ip6table_mangle 192 ip6table_raw 193 ip6table_filter 194 EOF 195 196 # Install CNI plugins 197 # - https://developer.hashicorp.com/nomad/docs/networking/cni 198 curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.5.0/cni-plugins-linux-arm64-v1.5.0.tgz" 199 mkdir -p /opt/cni/bin 200 tar -C /opt/cni/bin -xzf cni-plugins.tgz 201 202 # Prepare a Jenkins user for Jenkins agents and workload 203 # https://code.in0rdr.ch/nomad/file/docker/docker-jenkins-inbound-agent/README.html 204 groupadd -g $NOMAD_JENKINS_GID jenkins 205 useradd -m -s /bin/bash -u $NOMAD_JENKINS_UID -g $NOMAD_JENKINS_GID jenkins 206 207 # keep my podman.socket enabled even if no jenkins user is logged in 208 loginctl enable-linger jenkins 209 210 # https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#-M 211 systemctl --user -M jenkins@ start podman.socket 212 systemctl --user -M jenkins@ enable podman.socket 213 214 # create the mountpoint for the workspaces, podman does not create it for us 215 sudo -u jenkins mkdir /home/jenkins/workspace 216 217 # move Nomad server truststore 218 sudo mv $TLS_DIR/nomad-agent-ca.p12 /home/jenkins/ 219 sudo chown jenkins: /home/jenkins/nomad-agent-ca.p12 220 221 fi # endif NOMAD_CLIENT 222 223 # Configure .bashrc 224 cat << EOF >> "/home/${USERNAME}/.bashrc" 225 226 complete -C /usr/bin/nomad nomad 227 export NOMAD_ADDR=https://127.0.0.1:4646 228 export NOMAD_CACERT=$TLS_DIR/nomad-agent-ca.pem 229 export NOMAD_CLIENT_CERT=$TLS_DIR/dc1-cli-nomad.pem 230 export NOMAD_CLIENT_KEY=$TLS_DIR/dc1-cli-nomad-key.pem 231 EOF