commit cc2525a002adc4ee82626b09969aef4d354ab2ed
parent 80067c1f048292528a018bcee230fc8b5ac4a69b
Author: Andreas Gruhler <agruhl@gmx.ch>
Date: Sun, 12 Jan 2025 11:49:05 +0100
feat: qemu emulator on NixOS and Intel node
Diffstat:
6 files changed, 209 insertions(+), 55 deletions(-)
diff --git a/README.md b/README.md
@@ -1,8 +1,5 @@
# HashiPi
-
-A RaspberryPi test cluster for HashiCorp Nomad and OpenBao.
-
-
+A test cluster for HashiCorp Nomad and OpenBao.
## Client-Server Architecture
The cluster can be built with 3 or 5 nodes.
@@ -12,13 +9,13 @@ the Raft traffic minimal.
Example architecture with 5 RaspberyPi nodes:
-| Node | RAM | Nomad function | Bao function |
-|------|-----|----------------|--------------|
-| 00 | 4GB | client/server | server |
-| 01 | 4GB | client | n/a |
-| 02 | 4GB | client/server | server |
-| 03 | 4GB | client | n/a |
-| 04 | 4GB | client/server | server |
+| Node | Arch | RAM | Nomad function | Bao function |
+| ---- | ----- |-----|----------------|--------------|
+| 00 | arm64 | 4GB | client/server | server |
+| 01 | arm64 | 4GB | client | n/a |
+| 02 | arm64 | 4GB | client/server | server |
+| 03 | arm64 | 4GB | client | n/a |
+| 04 | arm64 | 4GB | client/server | server |
For best [performance on low power
devices](https://developer.hashicorp.com/consul/docs/install/performance), the
@@ -38,20 +35,21 @@ heartbeat_grace = "30s"
min_heartbeat_ttl = "30s"
```
-## ARM Builder
-These Packer files use the [Packer builder for
-ARM](https://github.com/mkaczanowski/packer-builder-arm). Follow the
-instructions in the repo to setup the builder and use the provided
-[examples](https://github.com/mkaczanowski/packer-builder-arm/tree/master/boards)
-to modify to your liking.
+## Packer Builders
+These Packer files use the following builders:
+* [Packer builder for ARM (plugin
+ "cross")](https://github.com/michalfita/packer-plugin-cross)
+* [QEMU](https://developer.hashicorp.com/packer/integrations/hashicorp/qemu)
+ builder
-## Supported Architectures
+The packer builder "cross" requires `qemu-img`.
-ARM >= ARMv7
+## Supported Architectures
+* ARM >= ARMv7
+* AMD64
## Self-signed TLS Certificates
### OpenBao
-
To create a new self-signed CA certificate for Bao:
```bash
# create bao self-signed CA certificate in ./tls/vault/
@@ -109,7 +107,6 @@ tbd
* ACLs for Nomad
## Nomad workload identity configuration
-
Follow along the tutorial to configure Nomad workload identities with Bao:
* https://developer.hashicorp.com/nomad/tutorials/integrate-vault/vault-acl
@@ -151,21 +148,10 @@ path "kv/+/{{identity.entity.aliases.AUTH_METHOD_ACCESSOR.metadata.nomad_job_id}
$ bao policy write nomad-workloads vault-policy-nomad-workloads.hcl
```
-
## Authorized Keys
Copy the contents of an openssh pubkey to `authorized_keys` Packer variable.
-## Run Packer
-Build [`packer-builder-arm`
-plugin](https://github.com/mkaczanowski/packer-builder-arm) according to
-projects instructions and move it to the working directory:
-```bash
-git clone git@github.com:mkaczanowski/packer-builder-arm.git arm-builder
-cd arm-builder
-go build
-cp packer-builder-arm .. && cd -
-```
-
+## Statically linked QEMU emulator
Install the statically linked qemu (emulator) binary. On Debian/Ubuntu:
```bash
sudo apt-get install qemu-user-static
@@ -208,15 +194,42 @@ interpreter /usr/bin/qemu-aarch64-static
...
```
+## QEMU emulator on NixOS
+On NixOS, it is sufficient to [enable the binfmt wrapper in the
+configuration](https://wiki.nixos.org/wiki/NixOS_on_ARM/Building_Images):
+```
+boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
+```
+
+The wrapper on NixOS can be checked on this path:
+```bash
+cat /proc/sys/fs/binfmt_misc/aarch64-linux
+```
+
+On NixOS, the interpreter is on this path (`hashi-pi.pkr.hcl`):
+```bash
+ "qemu_binary_source_path": "/run/binfmt/aarch64-linux",
+ "qemu_binary_destination_path": "/run/binfmt/aarch64-linux"
+```
+
+## Run Packer
+Initialize required packer plugins:
+```bash
+sudo packer init .
+```
+
Run packer with a value file to build an image for one host:
```bash
-sudo /usr/bin/packer build -var-file=variables.auto.pkrvars.hcl -var-file=hosts/pi0.pkrvars.hcl hashi-pi.pkr.hcl
+sudo packer build \
+ -only cross.hashipi \ # change to qemu.hashiintel to build amd images
+ -var-file=variables.auto.pkrvars.hcl \
+ -var-file=hosts/pi0.pkrvars.hcl \
+ hashi-pi.pkr.hcl
```
The `variable.auto.pkrvars.hcl` contains all sensitive packer variables.
## Write Image to SD Card
-
To [write the resulting image file to the sd
card](https://www.raspberrypi.org/documentation/installation/installing-images/linux.md)
with `dd`:
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -19,12 +19,6 @@ ff02::2 ip6-allrouters
127.0.1.1 $HOSTNAME
EOF
-# Install script to resize fs
-mv /tmp/resizerootfs.service /etc/systemd/system/
-chmod +x /tmp/resizerootfs
-mv /tmp/resizerootfs /usr/sbin/
-systemctl enable resizerootfs.service
-
# Add HashiCorp repository
wget -O- https://apt.releases.hashicorp.com/gpg \
| gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -33,7 +27,7 @@ echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://
# Install packages
apt-get update
-DEBIAN_FRONTEND=noninteractive apt-get install -y jq podman cloud-init \
+DEBIAN_FRONTEND=noninteractive apt-get install -y jq podman cloud-init curl \
"nomad=${NOMAD_VERSION}-1"
curl -L -o openbao.deb "https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}/bao_${BAO_VERSION}_linux_arm64.deb"
@@ -76,3 +70,18 @@ EOF
chown -R $USERNAME "/home/${USERNAME}/.ssh"
chmod 700 "/home/${USERNAME}/.ssh"
chmod 600 "/home/${USERNAME}/.ssh/authorized_keys"
+
+# Add cloud-init configuration for first boot
+# https://cloudinit.readthedocs.io/en/latest/reference/yaml_examples/disk_setup.html
+cat << EOF > /etc/cloud/cloud.cfg.d/99_hashipi_cloudinit.cfg
+#cloud-config
+
+# Growpart is enabled by default on the root partition
+#growpart:
+# mode: auto
+# devices: [\"/\"]
+# ignore_growroot_disabled: false
+
+# Resize filesystem to use all available space on partition
+resize_rootfs: noblock
+EOF
diff --git a/config/preseed.cfg b/config/preseed.cfg
@@ -0,0 +1,94 @@
+# Preseeding only locale sets language, country and locale.
+d-i debian-installer/locale string en_US
+
+# Keyboard selection.
+d-i console-setup/ask_detect boolean false
+d-i keyboard-configuration/xkb-keymap select us
+
+# Set the network interface
+d-i netcfg/choose_interface select enp0s2
+
+### Clock and time zone setup
+d-i clock-setup/utc boolean true
+d-i time/zone string UTC
+
+# Avoid that last message about the install being complete.
+d-i finish-install/reboot_in_progress note
+
+# This is fairly safe to set, it makes grub install automatically to the MBR
+# if no other operating system is detected on the machine.
+d-i grub-installer/only_debian boolean true
+# To install to the first device (assuming it is not a USB stick):
+d-i grub-installer/bootdev string default
+
+### Mirror settings
+# If you select ftp, the mirror/country string does not need to be set.
+#d-i mirror/protocol string ftp
+d-i mirror/country string manual
+d-i mirror/http/hostname string debian.ethz.ch
+d-i mirror/http/directory string /debian
+d-i mirror/http/proxy string
+
+### Apt setup
+# Select which update services to use; define the mirrors to be used.
+# Values shown below are the normal defaults.
+d-i apt-setup/services-select multiselect security, updates
+d-i apt-setup/security_host string security.debian.org
+
+# Do not prompt for scanning of additional CDs
+apt-cdrom-setup apt-setup/cdrom/set-first boolean false
+apt-cdrom-setup apt-setup/cdrom/set-next boolean false
+apt-cdrom-setup apt-setup/cdrom/set-failed boolean false
+apt-cdrom-setup apt-setup/cdrom/set-double boolean false
+
+# Don't report statistics
+popularity-contest popularity-contest/participate boolean false
+
+### Partitioning
+d-i partman-auto/method string lvm
+
+# This makes partman automatically partition without confirmation.
+d-i partman-partitioning/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+
+# Choose atomic partitioning recipes (all files in one partition)
+d-i partman-auto/choose_recipe select atomic
+# Use max space available for the new LVM volume
+d-i partman-auto-lvm/guided_size string max
+
+# Confirmation to write the lvm partitions
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+
+# Disable root account
+d-i passwd/root-login boolean true
+d-i passwd/root-password-crypted password !disabled
+d-i passwd/make-user boolean true
+
+### Account setup
+d-i passwd/username string in0rdr
+d-i passwd/user-fullname string in0rdr
+d-i passwd/user-uid string 1000
+d-i passwd/user-default-groups string sudo
+d-i passwd/user-password-crypted password !disabled
+#d-i passwd/user-password password pi
+#d-i passwd/user-password-again password pi
+
+# The installer will warn about weak passwords. If you are sure you know
+# what you're doing and want to override it, uncomment this.
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+
+### Package selection
+tasksel tasksel/first multiselect standard, ssh-server
+d-i pkgsel/include string openssh-server build-essential sudo
+d-i pkgsel/upgrade select full-upgrade
+
+# allow ssh root login
+#d-i preseed/late_command string \
+# in-target sed -i 's/^.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config
+d-i preseed/late_command string \
+ in-target sh -c 'mkdir -p /home/in0rdr/.ssh'; \
+ in-target sh -c 'echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2SnNAxEnre9hcPD74wNAouuXMgfIzwsB7qr88xSb8WS8CKqZGXzaQgebc0YExfV7PGyV6KUfu4KUvS1xDboRbU6ZLU4HdGlAi+hdv8dVVzdzCgFmdv5BEGam0SMhlzReWRiDvae0pObAPvAFg5ab6B/t1LjOosBOpPo2JfEkR6zfjDrMCYdEjWB5To1p5AX0BJneTiIeiEqR/05mZUk5L8hMFmwvm8QThd+SzpLY3zgWlWG7TlUQwx78xvell9KC0GChhwlkeEAwE3q1tq/LbgzvtY140Fg0bbBGcYQI4UvG85xfTfpbHeQ1RkSB8Rb8pMkaN7mT+3qhe08cHT9v3" > /home/in0rdr/.ssh/authorized_keys';
diff --git a/hashi-pi.pkr.hcl b/hashi-pi.pkr.hcl
@@ -1,3 +1,8 @@
+variable "manifest" {
+ type = string
+ default = "manifest.json"
+}
+
variable "authorized_keys" {
type = string
default = ""
@@ -134,7 +139,20 @@ variable "bao_version" {
default = "2.0.0-beta20240618"
}
-source "arm" "hashipi" {
+packer {
+ required_plugins {
+ qemu = {
+ version = ">= 1.1.0"
+ source = "github.com/hashicorp/qemu"
+ }
+ cross = {
+ version = ">= 1.1.3"
+ source = "github.com/michalfita/cross"
+ }
+ }
+}
+
+source "cross" "hashipi" {
file_checksum_type = "sha256"
file_checksum_url = "${var.img_url}.sha256"
file_target_extension = "xz"
@@ -165,18 +183,29 @@ source "arm" "hashipi" {
qemu_binary_source_path = "/usr/bin/qemu-aarch64-static"
}
-build {
- sources = ["source.arm.hashipi"]
-
- provisioner "file" {
- destination = "/tmp/resizerootfs"
- source = "arm-builder/scripts/resizerootfs/resizerootfs"
- }
+source "qemu" "hashiintel" {
+ accelerator = "kvm"
+ boot_command = ["<esc><wait>", "install <wait>", " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg <wait>", "debian-installer=en_US.UTF-8 <wait>", "auto <wait>", "locale=en_US.UTF-8 <wait>", "kbd-chooser/method=us <wait>", "keyboard-configuration/xkb-keymap=us <wait>", "netcfg/get_hostname={{ .Name }} <wait>", "netcfg/get_domain=libvirt <wait>", "fb=false <wait>", "debconf/frontend=noninteractive <wait>", "console-setup/ask_detect=false <wait>", "console-keymaps-at/keymap=us <wait>", "grub-installer/bootdev=/dev/sda <wait>", "<enter><wait>"]
+ boot_wait = "5s"
+ disk_interface = "virtio"
+ disk_size = "14000"
+ format = "qcow2"
+ # headless = true
+ http_directory = "config"
+ iso_checksum = "file:http://pkg.adfinis-on-exoscale.ch/debian/dists/Debian12.8/main/installer-amd64/20230607%2Bdeb12u8/images/MD5SUMS"
+ iso_url = "http://pkg.adfinis-on-exoscale.ch/debian/dists/Debian12.8/main/installer-amd64/20230607%2Bdeb12u8/images/netboot/mini.iso"
+ memory = "1024"
+ net_device = "virtio-net"
+ output_directory = "${var.hostname}"
+ shutdown_command = "echo 'debian' | sudo -S shutdown -P now"
+ ssh_timeout = "15m"
+ ssh_username = "in0rdr"
+ ssh_private_key_file = "/home/andi/.ssh/id_rsa"
+ vm_name = "${var.hostname}.qcow2"
+}
- provisioner "file" {
- destination = "/tmp/resizerootfs.service"
- source = "arm-builder/scripts/resizerootfs/resizerootfs.service"
- }
+build {
+ sources = ["source.cross.hashipi", "source.qemu.hashiintel"]
provisioner "shell" {
script = "bootstrap.sh"
@@ -236,7 +265,7 @@ build {
}
provisioner "shell" {
- script = "openbao.sh"
+ script = "openbao.sh"
remote_folder = "/home/${var.username}"
environment_vars = [
"USERNAME=${var.username}",
@@ -249,4 +278,9 @@ build {
"VAULT_TRANSIT_TOKEN=${var.vault_transit_token}"
]
}
+
+ post-processor "manifest" {
+ output = "${var.manifest}"
+ strip_path = true
+ }
}
diff --git a/hosts/intel0.pkrvars.hcl b/hosts/intel0.pkrvars.hcl
@@ -0,0 +1,4 @@
+hostname = "intel0"
+img_name = "HashiIntel0.img"
+nomad_server = true
+nomad_client = true
diff --git a/img/HashiPi-small.jpg b/img/HashiPi-small.jpg
Binary files differ.
