From 6c781a05039d50366617a4e553d6520374b2e923 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 11:27:52 -0300 Subject: [PATCH 01/17] clearer start-end of script --- workbench-script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workbench-script.py b/workbench-script.py index 02c1703..84653c2 100644 --- a/workbench-script.py +++ b/workbench-script.py @@ -273,7 +273,7 @@ def sync_time(): def main(): - print("START") + print("\n\nworkbench: START\n\n") parser=argparse.ArgumentParser() parser.add_argument("-p", "--path", required=True) parser.add_argument("-u", "--url", required=False) @@ -312,7 +312,7 @@ def main(): if args.url: send_snapshot_to_devicehub(snapshot, args.token, args.url) - print("END") + print("\n\nworkbench: END\n\n") if __name__ == '__main__': From d3018abde9b0744175bca3edf71e524ca6b84e8f Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 12:40:04 -0300 Subject: [PATCH 02/17] workbench: use sudo everywhere that means adjusting fine the hostname, which right now is hardcoded to workbench --- deploy-workbench.sh | 5 +++-- workbench-script.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 882e676..ef855cc 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -249,7 +249,7 @@ END2 ################### # configure hosts cat > /etc/hosts < /etc/hostname +echo workbench > /etc/hostname # check what linux images are available on the system # Figure out which Linux Kernel you want in the live environment. diff --git a/workbench-script.py b/workbench-script.py index 84653c2..ec0b459 100644 --- a/workbench-script.py +++ b/workbench-script.py @@ -228,9 +228,9 @@ def smartctl(all_disks, disk=None): def get_data(all_disks): - lshw = 'lshw -json' - hwinfo = 'hwinfo --reallyall' - dmidecode = 'dmidecode' + lshw = 'sudo lshw -json' + hwinfo = 'sudo hwinfo --reallyall' + dmidecode = 'sudo dmidecode' data = { 'lshw': exec_cmd(lshw), 'disks': smartctl(all_disks), From 8627cd80037caddd554675634c431523673327c4 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 12:40:49 -0300 Subject: [PATCH 03/17] bugifx workbench deploy use tee instead of cat here ${SUDO} is used, because it is assumed a third party system which could have its own configuration, so we deal the different situations fine (with or without sudo, etc.) --- deploy-workbench.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index ef855cc..bd9c8eb 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -198,8 +198,7 @@ create_persistence_partition() { mkdir -p "${tmp_rw_mount}" ${SUDO} mount "$(pwd)/${rw_img_path}" "${tmp_rw_mount}" ${SUDO} mkdir -p "${tmp_rw_mount}/settings" - # TODO without SUDO fails - ${SUDO} cat > "${tmp_rw_mount}/settings/settings.ini" < Date: Thu, 19 Sep 2024 12:42:03 -0300 Subject: [PATCH 04/17] workbench: use declarative config instead of args The only arg needed is the location of the config, which is optional --- .gitignore | 1 + deploy-workbench.sh | 2 +- workbench-script.py | 95 +++++++++++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index a454c06..9bd817e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ iso +settings.ini diff --git a/deploy-workbench.sh b/deploy-workbench.sh index bd9c8eb..8b13d04 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -271,7 +271,7 @@ stty -echo # Do not show what we type in terminal so it does not meddle with our dmesg -n 1 # Do not report *useless* system messages to the terminal # clearly specify the right working directory, used in the python script as os.getcwd() cd /mnt -pipenv run python /opt/workbench/workbench-script.py +pipenv run python /opt/workbench/workbench-script.py --config "/mnt/settings/settings.ini" stty echo END #TODO add some useful commands diff --git a/workbench-script.py b/workbench-script.py index ec0b459..be7de13 100644 --- a/workbench-script.py +++ b/workbench-script.py @@ -5,6 +5,7 @@ import json import uuid import hashlib import argparse +import configparser import ntplib import requests @@ -253,6 +254,7 @@ def save_snapshot_in_disk(snapshot, path): datetime.now().strftime("%Y%m%d-%H_%M_%S"), snapshot['uuid'] ) + print(f"workbench: Snapshot written in path '{filename}'") with open(filename, "w") as f: f.write(json.dumps(snapshot)) @@ -271,48 +273,75 @@ def sync_time(): ntplib.NTPClient() response = client.request('pool.ntp.org') +def load_config(config_file="settings.ini"): + """ + Tries to load configuration from a config file. + """ + config = configparser.ConfigParser() + + if os.path.exists(config_file): + # If config file exists, read from it + + print(f"workbench: Found config file in path: '{config_file}'.") + config.read(config_file) + path = config.get('settings', 'path', fallback=os.getcwd()) + # TODO validate that has http:// start + url = config.get('settings', 'url', fallback=None) + token = config.get('settings', 'token', fallback=None) + # TODO validate that the device exists? + device = config.get('settings', 'device', fallback=None) + erase = config.get('settings', 'erase', fallback=None) + else: + print(f"workbench: Config file '{config_file}' not found. Using default values.") + path = os.path.join(os.getcwd()) + url, token, device, erase = None, None, None, None + + return { + 'path': path, + 'url': url, + 'token': token, + 'device': device, + 'erase': erase + } + +def parse_args(): + """ + Parse config argument, if available + """ + parser = argparse.ArgumentParser(description="Optional config loader for workbench.") + parser.add_argument( + '--config', + help="Path to the config file. Defaults to 'settings.ini' in the current directory.", + default="settings.ini" # Fallback to 'settings.ini' by default + ) + return parser.parse_args() def main(): - print("\n\nworkbench: START\n\n") - parser=argparse.ArgumentParser() - parser.add_argument("-p", "--path", required=True) - parser.add_argument("-u", "--url", required=False) - parser.add_argument("-t", "--token", required=False) - parser.add_argument("-d", "--device", required=False) - parser.add_argument( - "-e", - "--erase", - choices=["basic", "baseline", "enhanced"], - required=False - ) - args=parser.parse_args() + vline='\n___________\n\n' + print(f"{vline}workbench: START\n") - if args.device and not args.erase: - print("error: argument --erase: expected one argument") - return - - if args.token and not args.url: - print("error: argument --url: expected one argument") - return - - if args.url and not args.token: - print("error: argument --token: expected one argument") - return + # Parse the command-line arguments + args = parse_args() + + # Load the config file, either specified via --config or the default 'settings.ini' + config_file = args.config + + config = load_config(config_file) all_disks = get_disks() snapshot = gen_snapshot(all_disks) - if args.erase and args.device: - snapshot['erase'] = gen_erase(all_disks, args.erase, user_disk=args.device) - elif args.erase: - snapshot['erase'] = gen_erase(all_disks, args.erase) + if config['erase'] and config['device']: + snapshot['erase'] = gen_erase(all_disks, config['erase'], user_disk=config['device']) + elif config['erase']: + snapshot['erase'] = gen_erase(all_disks, config['erase']) - save_snapshot_in_disk(snapshot, args.path) - - if args.url: - send_snapshot_to_devicehub(snapshot, args.token, args.url) + save_snapshot_in_disk(snapshot, config['path']) - print("\n\nworkbench: END\n\n") + if config['url']: + send_snapshot_to_devicehub(snapshot, config['token'], config['url']) + + print(f"\nworkbench: END{vline}") if __name__ == '__main__': From b005a5e158388ab25bd02df100b459d4db4db4f1 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 13:05:52 -0300 Subject: [PATCH 05/17] workbench deploy: bugfix workbench-script copy create directory and enforce that it is copied in directory --- deploy-workbench.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 8b13d04..a914ba8 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -260,8 +260,9 @@ prepare_app() { # prepare app during prepare_chroot_env # Install hardware_metadata module workbench_dir="${ISO_PATH}/chroot/opt/workbench" - ${SUDO} cp workbench-script.py "${workbench_dir}" - ${SUDO} cp requirements.txt "${workbench_dir}" + ${SUDO} mkdir -p "${workbench_dir}" + ${SUDO} cp workbench-script.py "${workbench_dir}/" + ${SUDO} cp requirements.txt "${workbench_dir}/" # startup script execution cat > "${ISO_PATH}/chroot/root/.profile" < Date: Thu, 19 Sep 2024 13:06:25 -0300 Subject: [PATCH 06/17] workbench deploy: detect ubuntu and guide user --- deploy-workbench.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index a914ba8..fc56408 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -390,6 +390,11 @@ prepare_chroot_env() { if [ -z "${VERSION_CODENAME:-}" ]; then . /etc/os-release echo "TAKING OS-RELEASE FILE" + if [ "${ID}" = "debian" ]; then + echo "ERROR: ubuntu detected, then you are enforced to specify debian variant" + echo " use for example \`VERSION_CODENAME='bookworm'\` or similar" + exit 1 + fi fi chroot_path="${ISO_PATH}/chroot" From 45464c49f02b0d71f807fb4336b149009109a7ad Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 13:08:57 -0300 Subject: [PATCH 07/17] Makefile: boot_iso: link to workbench_production --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0718097..8c0d811 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ install_dependencies: boot_iso: sudo qemu-system-x86_64 \ -enable-kvm -m 2G -vga qxl -netdev user,id=wan -device virtio-net,netdev=wan,id=nic1 \ - -drive format=raw,file=iso/workbench.iso,cache=none,if=virtio + -drive format=raw,file=iso/workbench_production.iso,cache=none,if=virtio # src https://www.ubuntubuzz.com/2021/04/how-to-boot-uefi-on-qemu.html # needs `sudo apt-get install ovmf` From 6d3d4e5c2a200b196757cd74527bb591e6806c2a Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 19 Sep 2024 14:01:54 -0300 Subject: [PATCH 08/17] make upload snapshot work --- settings.ini.example | 6 ++++++ workbench-script.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 settings.ini.example diff --git a/settings.ini.example b/settings.ini.example new file mode 100644 index 0000000..35fb89b --- /dev/null +++ b/settings.ini.example @@ -0,0 +1,6 @@ +[settings] +url = http://127.0.0.1:8000/api/snapshot/ +token = '1234' +# path = /path/to/save +# device = your_device_name +# # erase = basic diff --git a/workbench-script.py b/workbench-script.py index be7de13..9edb2e1 100644 --- a/workbench-script.py +++ b/workbench-script.py @@ -259,12 +259,15 @@ def save_snapshot_in_disk(snapshot, path): f.write(json.dumps(snapshot)) +# TODO sanitize url, if url is like this, it fails +# url = 'http://127.0.0.1:8000/api/snapshot/' def send_snapshot_to_devicehub(snapshot, token, url): headers = { f"Authorization": "Basic {token}", "Content-Type": "application/json" } - return requests.post(url, data=snapshot, header=headers) + + return requests.post(url, data=json.dumps(snapshot), headers=headers) @logs From d0f3f337d2e2944aafe6648fb62554cfc93c0dcf Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 11:36:24 -0300 Subject: [PATCH 09/17] deploy-workbench.sh: update with new settings.ini --- deploy-workbench.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index fc56408..17ac89f 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -201,12 +201,8 @@ create_persistence_partition() { ${SUDO} tee "${tmp_rw_mount}/settings/settings.ini" < Date: Fri, 20 Sep 2024 11:36:42 -0300 Subject: [PATCH 10/17] deploy-workbench.sh: use proper sudo here --- deploy-workbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 17ac89f..0fa487e 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -98,7 +98,7 @@ LABEL linux END )" # TIMEOUT 60 means 6 seconds :) - sudo tee "${ISO_PATH}/staging/isolinux/isolinux.cfg" < Date: Fri, 20 Sep 2024 11:39:23 -0300 Subject: [PATCH 11/17] deploy-workbench.sh: just copy local settings.ini --- deploy-workbench.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 0fa487e..039f4ef 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -198,12 +198,7 @@ create_persistence_partition() { mkdir -p "${tmp_rw_mount}" ${SUDO} mount "$(pwd)/${rw_img_path}" "${tmp_rw_mount}" ${SUDO} mkdir -p "${tmp_rw_mount}/settings" - ${SUDO} tee "${tmp_rw_mount}/settings/settings.ini" < Date: Fri, 20 Sep 2024 11:58:58 -0300 Subject: [PATCH 12/17] deploy-workbench.sh: add FORCE var to recreate persistence partition --- deploy-workbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 039f4ef..7a8a972 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -188,7 +188,7 @@ create_persistence_partition() { # persistent partition rw_img_name="workbench_vfat.img" rw_img_path="${ISO_PATH}/staging/${rw_img_name}" - if [ ! -f "${rw_img_path}" ] || [ "${DEBUG:-}" ]; then + if [ ! -f "${rw_img_path}" ] || [ "${DEBUG:-}" ] || [ "${FORCE:-}" ]; then ${SUDO} dd if=/dev/zero of="${rw_img_path}" bs=10M count=1 ${SUDO} mkfs.vfat "${rw_img_path}" From cb7ee619459c59e8a1f7409bf18cc3fb7daff64a Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 16:31:12 -0300 Subject: [PATCH 13/17] workbench-script: improve INFO and ERROR logs --- workbench-script.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/workbench-script.py b/workbench-script.py index 9edb2e1..df5c744 100644 --- a/workbench-script.py +++ b/workbench-script.py @@ -254,7 +254,7 @@ def save_snapshot_in_disk(snapshot, path): datetime.now().strftime("%Y%m%d-%H_%M_%S"), snapshot['uuid'] ) - print(f"workbench: Snapshot written in path '{filename}'") + print(f"workbench: INFO: Snapshot written in path '{filename}'") with open(filename, "w") as f: f.write(json.dumps(snapshot)) @@ -266,9 +266,11 @@ def send_snapshot_to_devicehub(snapshot, token, url): f"Authorization": "Basic {token}", "Content-Type": "application/json" } - - return requests.post(url, data=json.dumps(snapshot), headers=headers) - + try: + requests.post(url, data=json.dumps(snapshot), headers=headers) + print(f"workbench: INFO: Snapshot sent to '{url}'") + except: + print(f"workbench: ERROR: Snapshot not remotely sent. URL '{url}' is unreachable. Do you have internet? Is your server up & running?") @logs def sync_time(): @@ -285,7 +287,7 @@ def load_config(config_file="settings.ini"): if os.path.exists(config_file): # If config file exists, read from it - print(f"workbench: Found config file in path: '{config_file}'.") + print(f"workbench: INFO: Found config file in path: '{config_file}'.") config.read(config_file) path = config.get('settings', 'path', fallback=os.getcwd()) # TODO validate that has http:// start @@ -295,7 +297,7 @@ def load_config(config_file="settings.ini"): device = config.get('settings', 'device', fallback=None) erase = config.get('settings', 'erase', fallback=None) else: - print(f"workbench: Config file '{config_file}' not found. Using default values.") + print(f"workbench: ERROR: Config file '{config_file}' not found. Using default values.") path = os.path.join(os.getcwd()) url, token, device, erase = None, None, None, None From d53d8eb4d01ce5a33e343bf1996ac3409586c58f Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 16:31:25 -0300 Subject: [PATCH 14/17] .gitignore: ignore snapshots (*.json) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9bd817e..b9aa2fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ iso settings.ini +# ignore all possible snapshots in this dir +*.json From f1de55e0b3775c497228760bde49034f6a406188 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 16:51:48 -0300 Subject: [PATCH 15/17] deploy-workbench: increase persistence partition allow more snapshots on the persistence partition from 10 to 100 MiB --- deploy-workbench.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 7a8a972..00c83ec 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -189,7 +189,8 @@ create_persistence_partition() { rw_img_name="workbench_vfat.img" rw_img_path="${ISO_PATH}/staging/${rw_img_name}" if [ ! -f "${rw_img_path}" ] || [ "${DEBUG:-}" ] || [ "${FORCE:-}" ]; then - ${SUDO} dd if=/dev/zero of="${rw_img_path}" bs=10M count=1 + persistent_volume_size=100 + ${SUDO} dd if=/dev/zero of="${rw_img_path}" bs=1M count=${persistent_volume_size} ${SUDO} mkfs.vfat "${rw_img_path}" # generate structure on persistent partition From 8fc62ba8da80bf3337312d7d5c1e20f38835040e Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 17:03:50 -0300 Subject: [PATCH 16/17] deploy-workbench.sh: bugfix wrong check --- deploy-workbench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 00c83ec..9f1bcfc 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -382,7 +382,7 @@ prepare_chroot_env() { if [ -z "${VERSION_CODENAME:-}" ]; then . /etc/os-release echo "TAKING OS-RELEASE FILE" - if [ "${ID}" = "debian" ]; then + if [ ! "${ID}" = "debian" ]; then echo "ERROR: ubuntu detected, then you are enforced to specify debian variant" echo " use for example \`VERSION_CODENAME='bookworm'\` or similar" exit 1 From a47fab6c35b8be85c32a113537de75bc49737228 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 20 Sep 2024 19:55:21 -0300 Subject: [PATCH 17/17] deploy-workbench.sh: clearer persistence size --- deploy-workbench.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy-workbench.sh b/deploy-workbench.sh index 9f1bcfc..ffa4cdc 100755 --- a/deploy-workbench.sh +++ b/deploy-workbench.sh @@ -189,8 +189,8 @@ create_persistence_partition() { rw_img_name="workbench_vfat.img" rw_img_path="${ISO_PATH}/staging/${rw_img_name}" if [ ! -f "${rw_img_path}" ] || [ "${DEBUG:-}" ] || [ "${FORCE:-}" ]; then - persistent_volume_size=100 - ${SUDO} dd if=/dev/zero of="${rw_img_path}" bs=1M count=${persistent_volume_size} + persistent_volume_size_MB=100 + ${SUDO} dd if=/dev/zero of="${rw_img_path}" bs=1M count=${persistent_volume_size_MB} ${SUDO} mkfs.vfat "${rw_img_path}" # generate structure on persistent partition