This is part of a series of posts on the design and technical steps of creating Himblick, a digital signage box based on the Raspberry Pi 4.
In modern times, there are tools for provisioning systems that do useful things and allow to store an entire system configuration in text files committed to git. They are good in being able to reproducibly setup a system, and being able to inspect its contents from looking at the provisioning configuration instead of wading into it.
I normally use Ansible. It does have a chroot connector, but it has some serious limitations.
The biggest issue is that ansible's chroot connector does not mount /dev, /proc
and so on, which greatly limits what can be run inside it. Specifically,
installing many .deb packages will fail.
We work around it by copying Ansible needs inside the chroot (including Ansible itself), and then run it under systemd-nspawn using the local connector.
Systemd ansible operations still don't work
despite upstream having closed the issue,
so playbooks cannot use the systemd module.
Systemd is however able to perform operations inside the chroot using the
--root option, so we can do enabling and disabling in himblick and leave the
rest of the provisioning to Ansible:
def systemctl_enable(self, unit: str):
"""
Enable (and if needed unmask) the given systemd unit
"""
with self.working_resolvconf("/etc/resolv.conf"):
env = dict(os.environ)
env["LANG"] = "C"
subprocess.run(["systemctl", "--root=" + self.root, "enable", unit], check=True, env=env)
subprocess.run(["systemctl", "--root=" + self.root, "unmask", unit], check=True, env=env)
def systemctl_disable(self, unit: str, mask=True):
"""
Disable (and optionally mask) the given systemd unit
"""
with self.working_resolvconf("/etc/resolv.conf"):
env = dict(os.environ)
env["LANG"] = "C"
subprocess.run(["systemctl", "--root=" + self.root, "disable", unit], check=True, env=env)
if mask:
subprocess.run(["systemctl", "--root=" + self.root, "mask", unit], check=True, env=env)
This is the code that runs Ansible in the chroot. Leaving the playbook inside
/srv/himblick makes it possible to try tweaks in the running system during
development:
def run_ansible(self, playbook, roles, host_vars):
"""
Run ansible inside the chroot
"""
# Make sure ansible is installed in the chroot
self.apt_install("ansible")
# Create an ansible environment inside the rootfs
ansible_dir = self.abspath("/srv/himblick/ansible", create=True)
# Copy the ansible playbook and roles
self.copy_to("rootfs.yaml", "/srv/himblick/ansible")
self.copy_to("roles", "/srv/himblick/ansible")
# Write the variables
vars_file = os.path.join(ansible_dir, "himblick-vars.yaml")
with open(vars_file, "wt") as fd:
yaml.dump(host_vars, fd)
# Write ansible's inventory
ansible_inventory = os.path.join(ansible_dir, "inventory.ini")
with open(ansible_inventory, "wt") as fd:
print("[rootfs]", file=fd)
print("localhost ansible_connection=local", file=fd)
# Write ansible's config
ansible_cfg = os.path.join(ansible_dir, "ansible.cfg")
with open(ansible_cfg, "wt") as fd:
print("[defaults]", file=fd)
print("nocows = 1", file=fd)
print("inventory = inventory.ini", file=fd)
print("[inventory]", file=fd)
# See https://github.com/ansible/ansible/issues/48859
print("enable_plugins = ini", file=fd)
# Write ansible's startup script
args = ["exec", "ansible-playbook", "-v", "rootfs.yaml"]
ansible_sh = os.path.join(ansible_dir, "rootfs.sh")
with open(ansible_sh, "wt") as fd:
print("#!/bin/sh", file=fd)
print("set -xue", file=fd)
print('cd $(dirname -- "$0")', file=fd)
print("export ANSIBLE_CONFIG=ansible.cfg", file=fd)
print(" ".join(shlex.quote(x) for x in args), file=fd)
os.chmod(ansible_sh, 0o755)
# Run ansible in the chroot using systemd-nspawn
self.run(["/srv/himblick/ansible/rootfs.sh"], check=True)
def run(self, cmd: List[str], check=True, **kw) -> subprocess.CompletedProcess:
"""
Run the given command inside the chroot
"""
log.info("%s: running %s", self.root, " ".join(shlex.quote(x) for x in cmd))
chroot_cmd = ["systemd-nspawn", "-D", self.root]
chroot_cmd.extend(cmd)
if "env" not in kw:
kw["env"] = dict(os.environ)
kw["env"]["LANG"] = "C"
with self.working_resolvconf("/etc/resolv.conf"):
return subprocess.run(chroot_cmd, check=check, **kw)
Finally, to speed things up, here's a trick to cache the .deb files
downloaded during provisioning, and reuse them for the following runs:
def save_apt_cache(self, chroot: Chroot):
"""
Copy .deb files from the apt cache in the rootfs to our local cache
"""
if not self.cache:
return
rootfs_apt_cache = chroot.abspath("/var/cache/apt/archives")
apt_cache_root = self.cache.get("apt")
for fn in os.listdir(rootfs_apt_cache):
if not fn.endswith(".deb"):
continue
src = os.path.join(rootfs_apt_cache, fn)
dest = os.path.join(apt_cache_root, fn)
if os.path.exists(dest) and os.path.getsize(dest) == os.path.getsize(src):
continue
shutil.copy(src, dest)
def restore_apt_cache(self, chroot: Chroot):
"""
Copy .deb files from our local cache to the apt cache in the rootfs
"""
if not self.cache:
return
rootfs_apt_cache = chroot.abspath("/var/cache/apt/archives")
apt_cache_root = self.cache.get("apt")
for fn in os.listdir(apt_cache_root):
if not fn.endswith(".deb"):
continue
src = os.path.join(apt_cache_root, fn)
dest = os.path.join(rootfs_apt_cache, fn)
if os.path.exists(dest) and os.path.getsize(dest) == os.path.getsize(src):
continue
shutil.copy(src, dest)