commit bf302a44fb488a5be571cc97a75b9f1d2b974698
parent 6e9eccfc4ca6199e7ac80bf8fa3ce4a8f083de0e
Author: Andreas Gruhler <andreas.gruhler@adfinis.com>
Date: Wed, 19 Jul 2023 23:37:11 +0200
feat: add nomad/consul server/client flags
Diffstat:
8 files changed, 89 insertions(+), 61 deletions(-)
diff --git a/consul.sh b/consul.sh
@@ -60,6 +60,20 @@ performance {
}
EOF
+systemctl enable consul
+
+# Configure .bashrc
+cat << EOF >> .bashrc
+
+complete -C /usr/bin/consul consul
+
+export CONSUL_HTTP_ADDR="https://127.0.0.1:8501"
+export CONSUL_CACERT="/opt/consul/tls/consul-agent-ca.pem"
+export CONSUL_CLIENT_CERT="/opt/consul/tls/dc1-cli-consul.pem"
+export CONSUL_CLIENT_KEY="/opt/consul/tls/dc1-cli-consul-key.pem"
+EOF
+
+if [[ "$CONSUL_SERVER" = true ]]; then
cat << EOF > /etc/consul.d/server.hcl
server = true
bootstrap_expect = 3
@@ -85,16 +99,4 @@ tls {
}
}
EOF
-
-systemctl enable consul
-
-# Configure .bashrc
-cat << EOF >> .bashrc
-
-complete -C /usr/bin/consul consul
-
-export CONSUL_HTTP_ADDR="https://127.0.0.1:8501"
-export CONSUL_CACERT="/opt/consul/tls/consul-agent-ca.pem"
-export CONSUL_CLIENT_CERT="/opt/consul/tls/dc1-cli-consul.pem"
-export CONSUL_CLIENT_KEY="/opt/consul/tls/dc1-cli-consul-key.pem"
-EOF
+fi # endif CONSUL_SERVER
diff --git a/hashi-pi.json b/hashi-pi.json
@@ -8,11 +8,14 @@
"flash_device_path": null,
"img_size": "3G",
"consul_version": "1.15.3",
+ "consul_server": "true",
"consul_encrypt": null,
"consul_tls_ca": null,
"consul_tls_certs": null,
"consul_retry_join": "\"HashiPi0\", \"HashiPi1\", \"HashiPi2\"",
"nomad_version": "1.5.6",
+ "nomad_server": "true",
+ "nomad_client": "true",
"nomad_nfs_server": "",
"nomad_nfs_mount": "",
"nomad_nfs_target": "",
@@ -108,6 +111,7 @@
"remote_folder": "/home/{{ user `username` }}",
"environment_vars": [
"USERNAME={{ user `username` }}",
+ "CONSUL_SERVER={{ user `consul_server` }}",
"CONSUL_ENCRYPT={{ user `consul_encrypt` }}",
"CONSUL_RETRY_JOIN={{ user `consul_retry_join` }}"
]
@@ -122,6 +126,8 @@
"NFS_MOUNT={{ user `nomad_nfs_mount` }}",
"NFS_MOUNT_TARGET={{ user `nomad_nfs_target` }}",
"CILIUM_VERSION={{ user `cilium_version` }}",
+ "NOMAD_SERVER={{ user `nomad_server` }}",
+ "NOMAD_CLIENT={{ user `nomad_client` }}",
"NOMAD_PODMAN_DRIVER_VERSION={{ user `nomad_podman_driver_version` }}"
]
},
diff --git a/hosts/pi0.json b/hosts/pi0.json
@@ -3,10 +3,13 @@
"authorized_keys": "",
"img_name": "HashiPi-pi0.img",
"flash_device_path": "/dev/sda",
+ "consul_server": "true",
"consul_encrypt": "",
"consul_tls_ca": "./tls/consul/consul-agent-ca.pem",
"consul_tls_certs": "./tls/consul/certs/",
"consul_retry_join": "\"pi0.lan\", \"pi1.lan\", \"pi2.lan\", \"pi3.lan\", \"pi4.lan\"",
"vault_transit_server": "",
- "vault_transit_token": ""
+ "vault_transit_token": "",
+ "nomad_server": "true",
+ "nomad_client": "false"
}
diff --git a/hosts/pi1.json b/hosts/pi1.json
@@ -3,10 +3,13 @@
"authorized_keys": "",
"img_name": "HashiPi-pi1.img",
"flash_device_path": "/dev/sda",
+ "consul_server": "false",
"consul_encrypt": "",
"consul_tls_ca": "./tls/consul/consul-agent-ca.pem",
"consul_tls_certs": "./tls/consul/certs/",
"consul_retry_join": "\"pi0.lan\", \"pi1.lan\", \"pi2.lan\", \"pi3.lan\", \"pi4.lan\"",
"vault_transit_server": "",
- "vault_transit_token": ""
+ "vault_transit_token": "",
+ "nomad_server": "false",
+ "nomad_client": "true"
}
diff --git a/hosts/pi2.json b/hosts/pi2.json
@@ -3,10 +3,13 @@
"authorized_keys": "",
"img_name": "HashiPi-pi2.img",
"flash_device_path": "/dev/sda",
+ "consul_server": "true",
"consul_encrypt": "",
"consul_tls_ca": "./tls/consul/consul-agent-ca.pem",
"consul_tls_certs": "./tls/consul/certs/",
"consul_retry_join": "\"pi0.lan\", \"pi1.lan\", \"pi2.lan\", \"pi3.lan\", \"pi4.lan\"",
"vault_transit_server": "",
- "vault_transit_token": ""
+ "vault_transit_token": "",
+ "nomad_server": "true",
+ "nomad_client": "true"
}
diff --git a/hosts/pi3.json b/hosts/pi3.json
@@ -3,10 +3,13 @@
"authorized_keys": "",
"img_name": "HashiPi-pi3.img",
"flash_device_path": "/dev/sda",
+ "consul_server": "false",
"consul_encrypt": "",
"consul_tls_ca": "./tls/consul/consul-agent-ca.pem",
"consul_tls_certs": "./tls/consul/certs/",
"consul_retry_join": "\"pi0.lan\", \"pi1.lan\", \"pi2.lan\", \"pi3.lan\", \"pi4.lan\"",
"vault_transit_server": "",
- "vault_transit_token": ""
+ "vault_transit_token": "",
+ "nomad_server": "false",
+ "nomad_client": "true"
}
diff --git a/hosts/pi4.json b/hosts/pi4.json
@@ -3,10 +3,13 @@
"authorized_keys": "",
"img_name": "HashiPi-pi4.img",
"flash_device_path": "/dev/sda",
+ "consul_server": "true",
"consul_encrypt": "",
"consul_tls_ca": "./tls/consul/consul-agent-ca.pem",
"consul_tls_certs": "./tls/consul/certs/",
"consul_retry_join": "\"pi0.lan\", \"pi1.lan\", \"pi2.lan\", \"pi3.lan\", \"pi4.lan\"",
"vault_transit_server": "",
- "vault_transit_token": ""
+ "vault_transit_token": "",
+ "nomad_server": "true",
+ "nomad_client": "true"
}
diff --git a/nomad.sh b/nomad.sh
@@ -36,53 +36,13 @@ consul {
}
EOF
-# this instance acts as a Nomad client agent
-cat << EOF > /etc/nomad.d/client.hcl
-client {
- enabled = true
- # Allow jobs to use the loopback interface for exposing
- # private services only accessible from the client nodes
- host_network "private" {
- cidr = "127.0.0.1/32"
- }
-
- # Avoid resource contention between host/workload when oversubscribing
- # memory. Reserve a portion of the node's resources from receiving tasks.
- # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription
- reserved {
- # CPU to reserve, in MHz
- cpu = 1000
- # memory to reserve, in MB
- memory = 256
- }
-}
-
-plugin "nomad-driver-podman" {
- config {
- volumes {
- # Netreap Cilium operator connects to Cilium agent on the node through
- # Unix socket on shared host path volume
- # https://cosmonic.com/blog/engineering/netreap-a-practical-guide-to-running-cilium-in-nomad
- enabled = true
- }
- recover_stopped = false
- }
-}
-
-# Enable raw exec driver (jobs w/o isolation)
-plugin "raw_exec" {
- config {
- enabled = true
- }
-}
-EOF
-
-# ..and as a Nomad server agent
+# Configure as a Nomad server agent
#
# https://www.nomadproject.io/docs/configuration
# Note that it is strongly recommended not to operate a node as both client and server,
# although this is supported to simplify development and testing.
+if [[ "$NOMAD_SERVER" = true ]]; then
cat << EOF > /etc/nomad.d/server.hcl
server {
enabled = true
@@ -130,6 +90,7 @@ EOF
# Add VAULT_TOKEN placeholder, replace w/ real token
# https://www.nomadproject.io/docs/integrations/vault-integration
echo "VAULT_TOKEN=changeme" > /etc/nomad.d/nomad.env
+fi # endif NOMAD_SERVER
systemctl enable nomad
@@ -139,6 +100,49 @@ cat << EOF >> .bashrc
complete -C /usr/bin/nomad nomad
EOF
+if [[ "$NOMAD_CLIENT" = true ]]; then
+# this instance acts as a Nomad client agent
+cat << EOF > /etc/nomad.d/client.hcl
+client {
+ enabled = true
+
+ # Allow jobs to use the loopback interface for exposing
+ # private services only accessible from the client nodes
+ host_network "private" {
+ cidr = "127.0.0.1/32"
+ }
+
+ # Avoid resource contention between host/workload when oversubscribing
+ # memory. Reserve a portion of the node's resources from receiving tasks.
+ # https://developer.hashicorp.com/nomad/docs/job-specification/resources#memory-oversubscription
+ reserved {
+ # CPU to reserve, in MHz
+ cpu = 1000
+ # memory to reserve, in MB
+ memory = 256
+ }
+}
+
+plugin "nomad-driver-podman" {
+ config {
+ volumes {
+ # Netreap Cilium operator connects to Cilium agent on the node through
+ # Unix socket on shared host path volume
+ # https://cosmonic.com/blog/engineering/netreap-a-practical-guide-to-running-cilium-in-nomad
+ enabled = true
+ }
+ recover_stopped = false
+ }
+}
+
+# Enable raw exec driver (jobs w/o isolation)
+plugin "raw_exec" {
+ config {
+ enabled = true
+ }
+}
+EOF
+
# Mount NFS for hostpath volumes
if [[ -n "$NFS_SERVER" ]]; then
mkdir -p "$NFS_MOUNT_TARGET"
@@ -146,7 +150,7 @@ if [[ -n "$NFS_SERVER" ]]; then
cat << EOF >> /etc/fstab
$NFS_SERVER:/$NFS_MOUNT $NFS_MOUNT_TARGET nfs defaults 0 0
EOF
-fi
+fi # endif NFS_SERVER
# Enable memory cgroups for NFS CSI
# https://downey.io/blog/exploring-cgroups-raspberry-pi
@@ -272,3 +276,4 @@ EOF
curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz"
mkdir -p /opt/cni/bin
tar -C /opt/cni/bin -xzf cni-plugins.tgz
+fi # endif NOMAD_CLIENT
