Merge pull request 'feature: workbench pxe' (#2) from pxe into main

Reviewed-on: #2

.gitignore

@@ -1,4 +1,4 @@
 iso
 settings.ini
-# ignore all possible snapshots in this dir
+# ignore all possible snapshots in this repo
 *.json

README-es.md

@@ -54,3 +54,9 @@ OPCIONES
 ## Enfoque
 
 workbench-script trata de ser simple y minimalista, una función principal y funciones de soporte la lectura de las diferentes funcionalidades.
+
+## Generar ISO para el USB
+
+Para generar la iso y preparar un usb que arranque con workbench necesitas generarte una workbench de este, con tu configuración específica
+
+Ejecuta `./deploy-workbench.sh`

deploy-workbench.sh

@@ -34,12 +34,12 @@ END
 
 create_iso() {
         # Copy kernel and initramfs
-        vmlinuz="$(ls -1v ${ISO_PATH}/chroot/boot/vmlinuz-* | tail -n 1)"
+        vmlinuz="$(ls -1v "${ISO_PATH}"/chroot/boot/vmlinuz-* | tail -n 1)"
-        initrd="$(ls -1v ${ISO_PATH}/chroot/boot/initrd.img-* | tail -n 1)"
+        initrd="$(ls -1v "${ISO_PATH}"/chroot/boot/initrd.img-* | tail -n 1)"
-        ${SUDO} cp ${vmlinuz} ${ISO_PATH}/staging/live/vmlinuz
+        ${SUDO} cp ${vmlinuz} "${ISO_PATH}"/staging/live/vmlinuz
-        ${SUDO} cp ${initrd} ${ISO_PATH}/staging/live/initrd
+        ${SUDO} cp ${initrd} "${ISO_PATH}"/staging/live/initrd
         # Creating ISO
-        iso_path="${ISO_PATH}/${iso_name}.iso"
+        iso_path=""${ISO_PATH}"/${iso_name}.iso"
 
         # 0x14 is FAT16 Hidden FAT16 <32, this is the only format detected in windows10 automatically when using a persistent volume of 10 MB
         ${SUDO} xorrisofs \
@@ -59,7 +59,7 @@ create_iso() {
                 -e /EFI/boot/efiboot.img \
                 -no-emul-boot \
                 -isohybrid-gpt-basdat \
-                -append_partition 2 0xef ${ISO_PATH}/staging/EFI/boot/efiboot.img \
+                -append_partition 2 0xef "${ISO_PATH}"/staging/EFI/boot/efiboot.img \
                 -append_partition 3 0x14 "${rw_img_path}" \
                 "${ISO_PATH}/staging"
 
@@ -138,7 +138,7 @@ EOF
 
         ${SUDO} grub-mkstandalone \
                 --format=x86_64-efi \
-                --output=${ISO_PATH}/tmp/bootx64.efi \
+                --output="${ISO_PATH}"/tmp/bootx64.efi \
                 --locales="" \
                 --fonts="" \
                 "boot/grub/grub.cfg=${ISO_PATH}/tmp/grub-standalone.cfg"
@@ -149,10 +149,10 @@ EOF
   #   grubx64 looks for a file in /EFI/debian/grub.cfg -> src src https://unix.stackexchange.com/questions/648089/uefi-grub-not-finding-config-file
         ${SUDO} cp /usr/lib/shim/shimx64.efi.signed /tmp/bootx64.efi
         ${SUDO} cp /usr/lib/grub/x86_64-efi-signed/grubx64.efi.signed /tmp/grubx64.efi
-        ${SUDO} cp ${ISO_PATH}/tmp/grub-standalone.cfg ${ISO_PATH}/staging/EFI/debian/grub.cfg
+        ${SUDO} cp "${ISO_PATH}/tmp/grub-standalone.cfg" "${ISO_PATH}/staging/EFI/debian/grub.cfg"
 
         (
-                cd ${ISO_PATH}/staging/EFI/boot
+                cd "${ISO_PATH}/staging/EFI/boot"
                 ${SUDO} dd if=/dev/zero of=efiboot.img bs=1M count=20
                 ${SUDO} mkfs.vfat efiboot.img
                 ${SUDO} mmd -i efiboot.img efi efi/boot
@@ -178,8 +178,8 @@ compress_chroot_dir() {
         # why squashfs -> https://unix.stackexchange.com/questions/163190/why-do-liveusbs-use-squashfs-and-similar-file-systems
         # noappend option needed to avoid this situation -> https://unix.stackexchange.com/questions/80447/merging-preexisting-source-folders-in-mksquashfs
         ${SUDO} mksquashfs \
-                ${ISO_PATH}/chroot \
+                "${ISO_PATH}/chroot" \
-                ${ISO_PATH}/staging/live/filesystem.squashfs \
+                "${ISO_PATH}/staging/live/filesystem.squashfs" \
                 ${DEBUG_SQUASHFS_ARGS:-} \
                 -noappend -e boot
 }
@@ -198,8 +198,13 @@ create_persistence_partition() {
                 ${SUDO} umount -f -l "${tmp_rw_mount}" >/dev/null 2>&1 || true
                 mkdir -p "${tmp_rw_mount}"
                 ${SUDO} mount "$(pwd)/${rw_img_path}" "${tmp_rw_mount}"
-                ${SUDO} mkdir -p "${tmp_rw_mount}/settings"
+                ${SUDO} mkdir -p "${tmp_rw_mount}"
-                ${SUDO} cp -v settings.ini "${tmp_rw_mount}/settings/settings.ini"
+                if [ -f "settings.ini" ]; then
+                        ${SUDO} cp -v settings.ini "${tmp_rw_mount}/settings.ini"
+                else
+                        echo "ERROR: settings.ini does not exist yet, cannot read config from there. You can take inspiration with file settings.ini.example"
+                        exit 1
+                fi
                 ${SUDO} umount "${tmp_rw_mount}"
 
                 uuid="$(blkid "${rw_img_path}" | awk '{ print $3; }')"
@@ -258,14 +263,32 @@ prepare_app() {
 
         # startup script execution
         cat > "${ISO_PATH}/chroot/root/.profile" <<END
-set -x
+if [ -f /tmp/workbench_lock ]; then
+        return 0
+else
+        touch /tmp/workbench_lock
+fi
 
+set -x
 stty -echo # Do not show what we type in terminal so it does not meddle with our nice output
 dmesg -n 1 # Do not report *useless* system messages to the terminal
+
+# detect pxe env
+nfs_host="\$(df -hT | grep nfs | cut -f1 -d: | head -n1)"
+if [ "\${nfs_host}" ]; then
+        mount --bind /run/live/medium /mnt
+        # debian live nfs path is readonly, do a trick
+        #   to make snapshots subdir readwrite
+        mount \${nfs_host}:/snapshots /run/live/medium/snapshots
+        # reload mounts on systemd
+        systemctl daemon-reload
+fi
 # clearly specify the right working directory, used in the python script as os.getcwd()
 cd /mnt
-pipenv run python /opt/workbench/workbench-script.py --config "/mnt/settings/settings.ini"
+pipenv run python /opt/workbench/workbench-script.py --config /mnt/settings.ini
+
 stty echo
+set +x
 END
         #TODO add some useful commands
         cat > "${ISO_PATH}/chroot/root/.bash_history" <<END
@@ -280,9 +303,7 @@ echo 'Install requirements'
 apt-get install -y --no-install-recommends \
   sudo \
   python3 python3-dev python3-pip pipenv \
-  dmidecode smartmontools hwinfo pciutils lshw < /dev/null
+  dmidecode smartmontools hwinfo pciutils lshw nfs-common < /dev/null
-  # Install python requirements using apt instead of pip
-  #python3-dateutils python3-decouple python3-colorlog
 
 # Install lshw B02.19 utility using backports (DEPRECATED in Debian 12)
 #apt install -y -t ${VERSION_CODENAME}-backports lshw  < /dev/null
@@ -405,7 +426,8 @@ install_requirements() {
         image_deps='debootstrap
                     squashfs-tools
                     xorriso
-                    mtools'
+                    mtools
+                    dosfstools'
         # secureboot:
         #   -> extra src https://wiki.debian.org/SecureBoot/
         #   -> extra src https://wiki.debian.org/SecureBoot/VirtualMachine
@@ -423,17 +445,17 @@ install_requirements() {
 
 # thanks https://willhaley.com/blog/custom-debian-live-environment/
 create_base_dirs() {
-        mkdir -p ${ISO_PATH}
+        mkdir -p "${ISO_PATH}"
-        mkdir -p ${ISO_PATH}/staging/EFI/boot
+        mkdir -p "${ISO_PATH}/staging/EFI/boot"
-        mkdir -p ${ISO_PATH}/staging/boot/grub/x86_64-efi
+        mkdir -p "${ISO_PATH}/staging/boot/grub/x86_64-efi"
-        mkdir -p ${ISO_PATH}/staging/isolinux
+        mkdir -p "${ISO_PATH}/staging/isolinux"
-        mkdir -p ${ISO_PATH}/staging/live
+        mkdir -p "${ISO_PATH}/staging/live"
-        mkdir -p ${ISO_PATH}/tmp
+        mkdir -p "${ISO_PATH}/tmp"
         # usb name
-        ${SUDO} touch ${ISO_PATH}/staging/${iso_name}
+        ${SUDO} touch "${ISO_PATH}/staging/${iso_name}"
 
         # for uefi secure boot grub config file
-        mkdir -p ${ISO_PATH}/staging/EFI/debian
+        mkdir -p "${ISO_PATH}/staging/EFI/debian"
 }
 
 # this function is used both in shell and chroot

pxe/.env.example

@@ -0,0 +1,4 @@
+server_ip=192.168.1.2
+nfs_allowed_lan=192.168.1.0/24
+tftp_path='/srv/pxe-tftp'
+nfs_path='/srv/pxe-nfs'

pxe/.gitignore

@@ -0,0 +1,2 @@
+.env
+pxe-menu.cfg

pxe/Makefile

@@ -0,0 +1,2 @@
+test_pxe:
+	qemu-system-x86_64 -m 1G -boot n -netdev user,id=mynet0,tftp=/srv/pxe-tftp,bootfile=pxelinux.0 -device e1000,netdev=mynet0

pxe/README-en.md

@@ -0,0 +1 @@
+TODO

pxe/README-es.md

@@ -0,0 +1,55 @@
+# workbench via PXE
+
+## Introducción
+
+Permite arrancar workbench a través de la red en vez de por USB. Utiliza la misma imagen generada por el script [deploy-workbench.sh](../deploy-workbench.sh), pero en el formato compatible con el arranque por red en vez de la iso.
+
+Ejecuta el siguiente script en un servidor debian estable que estará dedicado a la gestión del pxe server
+
+```
+./install-pxe.sh
+```
+
+Este servidor aporta un servicio de arranque por red tipo PXE, y no hace colisión con un servidor DHCP existente.
+
+## Funcionamiento
+
+El servidor PXE ofrece a la máquina que arranca un *debian live* a través de [NFS](https://es.wikipedia.org/wiki/Network_File_System). Una vez arrancado, ejecuta el `workbench-script.py` con la configuración remota del servidor PXE. Cuando ha terminado, también guarda en el mismo servidor PXE el snapshot resultante. También lo puede guardar en devicehub si se especifica en la variable `url` de la configuración `settings.ini`.
+
+## Probarlo todo en localhost
+
+Preparar configuración de `.env` tal como:
+
+```
+server_ip=10.0.2.2
+nfs_allowed_lan=10.0.2.0/24
+tftp_path='/srv/pxe-tftp'
+nfs_path='/srv/pxe-nfs'
+```
+
+Red y host 10.0.2.2? Esta es la forma en que el programa *qemu* hace red en localhost, 10.0.2.2 es la dirección de localhost que saliendo de qemu es traducida como 127.0.0.1
+
+Desplegar servidores TFTP y NFS en el mismo ordenador, para permitir nfs inseguro:
+
+```
+DEBUG=true ./install-pxe.sh
+```
+
+Los directorios inseguros contienen configuración y snapshots de workbench, nada importante supongo. Aún así, `DEBUG=true` no se recomienda para un entorno de producción para evitar sorpresas.
+
+
+Y para terminar, probar el cliente PXE con el siguiente comando:
+
+```
+make test_pxe
+```
+
+## Recursos
+
+El servicio PXE
+
+- Originalmente inspirado en este artículo https://farga.exo.cat/exo/wiki/src/branch/master/howto/apu/apu-installer.md
+- https://github.com/eReuse/workbench-live/blob/feature/pxe/docs/PXE-setup.md
+- https://wiki.debian.org/PXEBootInstall
+- https://wiki.debian.org/DebianInstaller/NetbootFirmware
+- [In this presentation](https://people.debian.org/~andi/LiveNetboot.pdf), recomienda página 12 [4.6 Building a netboot image](https://live-team.pages.debian.net/live-manual/html/live-manual/the-basics.en.html#236) [4.7 Webbooting](https://live-team.pages.debian.net/live-manual/html/live-manual/the-basics.en.html#275)

pxe/README.md

@@ -0,0 +1,5 @@
+# pxe
+
+- [Español](./README-es.md)
+- [English](./README-en.md)
+

pxe/install-pxe.sh

@@ -0,0 +1,158 @@
+#!/bin/sh
+
+# Copyright (c) 2024 Pedro <copyright@cas.cat>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+set -e
+set -u
+# DEBUG
+set -x
+
+detect_user() {
+        userid="$(id -u)"
+        # detect non root user without sudo
+        if [ ! "${userid}" = 0 ] && id ${USER} | grep -qv sudo; then
+                echo "ERROR: this script needs root or sudo permissions (current user is not part of sudo group)"
+                exit 1
+                # detect user with sudo or already on sudo src https://serverfault.com/questions/568627/can-a-program-tell-it-is-being-run-under-sudo/568628#568628
+        elif [ ! "${userid}" = 0 ] || [ -n "${SUDO_USER}" ]; then
+                SUDO='sudo'
+                # jump to current dir where the script is so relative links work
+                cd "$(dirname "${0}")"
+                # working directory to build the iso
+                ISO_PATH="iso"
+                # detect pure root
+        elif [ "${userid}" = 0 ]; then
+                SUDO=''
+                ISO_PATH="/opt/workbench"
+        fi
+}
+
+install_dependencies() {
+        ${SUDO} apt update
+        ${SUDO} apt install -y wget dnsmasq nfs-kernel-server rsync syslinux
+}
+
+backup_file() {
+        target="${1}"
+        ts="$(date +'%Y-%m-%d_%H-%M-%S')"
+
+        if [ -f "${target}" ]; then
+                if ! grep -q 'we should do a backup' "${target}"; then
+                        ${SUDO} cp -a "${target}" "${target}-bak_${ts}"
+                fi
+        fi
+}
+
+install_nfs() {
+        # append live directory, which is expected by the debian live env
+        ${SUDO} mkdir -p "${nfs_path}/live"
+        ${SUDO} mkdir -p "${nfs_path}/snapshots"
+
+        # debian live nfs path is readonly, do a trick
+        #   to make snapshots subdir readwrite
+        if ! grep -q "/snapshots" /proc/mounts; then
+                ${SUDO} mkdir -p "/snapshots"
+                ${SUDO} mount --bind "${nfs_path}/snapshots" "/snapshots"
+        fi
+
+        backup_file /etc/exports
+
+        if [ "${DEBUG:-}" ]; then
+                nfs_debug=' 127.0.0.1(rw,sync,no_subtree_check,no_root_squash,insecure)'
+        fi
+
+        ${SUDO} tee /etc/exports <<END
+${script_header}
+#   we assume that if you remove this line from the file, we should do a backup
+${nfs_path} ${nfs_allowed_lan}(rw,sync,no_subtree_check,no_root_squash)${nfs_debug:-}
+/snapshots ${nfs_allowed_lan}(rw,sync,no_subtree_check,no_root_squash)${nfs_debug:-}
+END
+        # reload nfs exports
+        ${SUDO} exportfs -vra
+
+
+        if [ ! -f "${nfs_path}/settings.ini" ]; then
+                if [ -f "settings.ini" ]; then
+                        ${SUDO} cp settings.ini "${nfs_path}/settings.ini"
+                else
+                        echo "ERROR: $(pwd)/settings.ini does not exist yet, cannot read config from there. You can take inspiration with file $(pwd)/settings.ini.example"
+                        exit 1
+                fi
+        fi
+}
+
+install_tftp() {
+
+        # from https://wiki.debian.org/PXEBootInstall#Simple_way_-_using_Dnsmasq
+        ${SUDO} tee /etc/dnsmasq.d/pxe-tftp <<END
+${script_header}
+port=0
+# info: https://wiki.archlinux.org/title/Dnsmasq#Proxy_DHCP
+dhcp-range=${nfs_allowed_lan%/*},proxy
+dhcp-boot=pxelinux.0
+pxe-service=x86PC,"Network Boot",pxelinux
+enable-tftp
+tftp-root=${tftp_path}
+END
+}
+
+install_netboot() {
+        # if you want to refresh install, remove or move dir
+        if [ ! -d "${tftp_path}" ] || [ "${FORCE:-}" ]; then
+                ${SUDO} mkdir -p "${tftp_path}/pxelinux.cfg"
+                if [ ! -f "${tftp_path}/netboot.tar.gz" ]; then
+                        url="http://ftp.debian.org/debian/dists/${VERSION_CODENAME}/main/installer-amd64/current/images/netboot/netboot.tar.gz"
+                        ${SUDO} wget -P "${tftp_path}" "${url}"
+                        ${SUDO} tar xvf "${tftp_path}/netboot.tar.gz" -C "${tftp_path}"
+                        ${SUDO} rm -rf "${tftp_path}/pxelinux.cfg"
+                        ${SUDO} mkdir -p "${tftp_path}/pxelinux.cfg"
+                fi
+
+                ${SUDO} cp -fv "${PXE_DIR}/../iso/staging/live/vmlinuz" "${tftp_path}/"
+                ${SUDO} cp -fv "${PXE_DIR}/../iso/staging/live/initrd" "${tftp_path}/"
+
+                ${SUDO} cp /usr/lib/syslinux/memdisk "${tftp_path}/"
+                ${SUDO} cp /usr/lib/syslinux/modules/bios/* "${tftp_path}/"
+                envsubst < ./pxe-menu.cfg | ${SUDO} tee "${tftp_path}/pxelinux.cfg/default"
+        fi
+
+        ${SUDO} rsync -av "${PXE_DIR}/../iso/staging/live/filesystem.squashfs" "${nfs_path}/live/"
+}
+
+init_config() {
+
+        # get where the script is
+        cd "$(dirname "${0}")"
+
+        # this is what we put in the files we modity
+        script_header='# configuration done through workbench install-pxe script'
+
+        PXE_DIR="$(pwd)"
+
+        if [ -f ./.env ]; then
+                . ./.env
+        else
+                echo "PXE: WARNING: $(pwd)/.env does not exist yet, cannot read config from there. You can take inspiration with file $(pwd)/.env.example"
+        fi
+        VERSION_CODENAME="${VERSION_CODENAME:-bookworm}"
+        tftp_path="${tftp_path:-/srv/pxe-tftp}"
+        # vars used in envsubst require to be exported:
+        export server_ip="${server_ip}"
+        export nfs_path="${nfs_path:-/srv/pxe-nfs}"
+}
+
+main() {
+        detect_user
+        init_config
+        install_dependencies
+        install_tftp
+        install_nfs
+        install_netboot
+        echo "PXE: Installation finished"
+}
+
+main "${@}"
+
+# written in emacs
+# -*- mode: shell-script; -*-

pxe/pxe-menu.cfg.example

@@ -0,0 +1,12 @@
+DEFAULT menu.c32
+PROMPT 0
+TIMEOUT 50
+ONTIMEOUT wb
+
+MENU TITLE PXE Boot Menu
+
+LABEL wb
+        MENU LABEL Boot Workbench
+        KERNEL vmlinuz
+        INITRD initrd
+        APPEND ip=dhcp netboot=nfs nfsroot=${server_ip}:${nfs_path}/ boot=live text forcepae

pxe/settings.ini.example

@@ -0,0 +1,6 @@
+[settings]
+url = http://localhost:8000/api/snapshot/
+token = '1234'
+path = /mnt
+# device = your_device_name
+# # erase = basic

settings.ini.example

@@ -1,5 +1,5 @@
 [settings]
-url = http://127.0.0.1:8000/api/snapshot/
+url = http://localhost:8000/api/snapshot/
 token = '1234'
 # path = /path/to/save
 # device = your_device_name

snapshots/README.md

@@ -0,0 +1,3 @@
+This is the path by default used by workbench-script
+
+You can change it in the configuration

workbench-script.py

@@ -7,7 +7,6 @@ import hashlib
 import argparse
 import configparser
 
-import ntplib
 import requests
 
 
@@ -228,6 +227,8 @@ def smartctl(all_disks, disk=None):
 ## End Command Functions ##
 
 
+# TODO permitir selección
+# TODO permitir que vaya más rápido
 def get_data(all_disks):
     lshw = 'sudo lshw -json'
     hwinfo = 'sudo hwinfo --reallyall'
@@ -249,15 +250,33 @@ def gen_snapshot(all_disks):
 
 
 def save_snapshot_in_disk(snapshot, path):
-    filename = "{}/{}_{}.json".format(
+    snapshot_path = os.path.join(path, 'snapshots')
-        path,
-        datetime.now().strftime("%Y%m%d-%H_%M_%S"),
-        snapshot['uuid']
-    )
-    print(f"workbench: INFO: Snapshot written in path '{filename}'")
-    with open(filename, "w") as f:
-        f.write(json.dumps(snapshot))
 
+    filename = "{}/{}_{}.json".format(
+        snapshot_path,
+        datetime.now().strftime("%Y%m%d-%H_%M_%S"),
+        snapshot['uuid'])
+
+    try:
+        if not os.path.exists(snapshot_path):
+            os.makedirs(snapshot_path)
+            print(f"workbench: INFO: Created snapshots directory at '{snapshot_path}'")
+
+        with open(filename, "w") as f:
+            f.write(json.dumps(snapshot))
+        print(f"workbench: INFO: Snapshot written in path '{filename}'")
+    except Exception as e:
+        try:
+            print(f"workbench: WARNING: Attempting to save in actual path. Reason: Failed to write in snapshots directory:\n    {e}.")
+            fallback_filename = "{}/{}_{}.json".format(
+                path,
+                datetime.now().strftime("%Y%m%d-%H_%M_%S"),
+                snapshot['uuid'])
+            with open(fallback_filename, "w") as f:
+                f.write(json.dumps(snapshot))
+                print(f"workbench: INFO: Snapshot written in fallback path '{fallback_filename}'")
+        except Exception as e:
+            print(f"workbench: ERROR: Could not save snapshot locally. Reason: Failed to write in fallback path:\n    {e}")
 
 # TODO sanitize url, if url is like this, it fails
 #   url = 'http://127.0.0.1:8000/api/snapshot/'
@@ -269,14 +288,8 @@ def send_snapshot_to_devicehub(snapshot, token, url):
     try:
         requests.post(url, data=json.dumps(snapshot), headers=headers)
         print(f"workbench: INFO: Snapshot sent to '{url}'")
-    except:
+    except Exception as e:
-        print(f"workbench: ERROR: Snapshot not remotely sent. URL '{url}' is unreachable. Do you have internet? Is your server up & running?")
+        print(f"workbench: ERROR: Snapshot not remotely sent. URL '{url}' is unreachable. Do you have internet? Is your server up & running?\n    {e}")
-
-@logs
-def sync_time():
-    # is neccessary?
-    ntplib.NTPClient()
-    response = client.request('pool.ntp.org')
 
 def load_config(config_file="settings.ini"):
     """
@@ -333,6 +346,11 @@ def main():
 
     config = load_config(config_file)
 
+    # TODO show warning if non root, means data is not complete
+    #   if annotate as potentially invalid snapshot (pending the new API to be done)
+    if os.geteuid() != 0:
+        print("workbench: WARNING: This script must be run as root. Collected data will be incomplete or unusable")
+
     all_disks = get_disks()
     snapshot = gen_snapshot(all_disks)