From ccb0be1b4c53b3b5a4c5d8e85809e88d7ea5807d Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Sat, 9 Nov 2024 16:30:49 -0300 Subject: [PATCH 01/12] check for values on websnapshot --- device/templates/details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device/templates/details.html b/device/templates/details.html index 331c857..137a4f6 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -58,7 +58,7 @@
{{ object.type }}
- {% if object.is_websnapshot %} + {% if object.is_websnapshot and object.last_user_evidence %} {% for k, v in object.last_user_evidence %}
{{ k }}
From 83dd044325f969c1bb22f7f1fd0d9a70256def92 Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Sat, 9 Nov 2024 16:46:01 -0300 Subject: [PATCH 02/12] better error handling and logging --- evidence/management/commands/up_snapshots.py | 25 ++++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/evidence/management/commands/up_snapshots.py b/evidence/management/commands/up_snapshots.py index c987d76..5f699a7 100644 --- a/evidence/management/commands/up_snapshots.py +++ b/evidence/management/commands/up_snapshots.py @@ -47,10 +47,23 @@ class Command(BaseCommand): self.open(filepath) def open(self, filepath): - with open(filepath, 'r') as file: - content = json.loads(file.read()) - path_name = save_in_disk(content, self.user.institution.name) - self.snapshots.append((content, path_name)) + try: + with open(filepath, 'r') as file: + content = json.loads(file.read()) + path_name = save_in_disk(content, self.user.institution.name) + + self.snapshots.append((content, path_name)) + + except json.JSONDecodeError as e: + logger.error("JSON decode error in file %s: %s", filepath, e) + raise ValueError(f"Invalid JSON format in file {filepath}") from e + except FileNotFoundError as e: + logger.error("File not found: %s", filepath) + raise FileNotFoundError(f"File not found: {filepath}") from e + #or we cath'em all + except Exception as e: + logger.exception("Unexpected error when opening file %s: %s", filepath, e) + raise Exception(f"Unexpected error when opening file {filepath}") from e def parsing(self): for s, p in self.snapshots: @@ -58,6 +71,8 @@ class Command(BaseCommand): self.devices.append(Build(s, self.user)) move_json(p, self.user.institution.name) except Exception as err: + if settings.DEBUG: + logger.exception("%s", err) snapshot_id = s.get("uuid", "") - txt = "Could not parse snapshot: %s" + txt = "It is not possible to parse snapshot: %s" logger.error(txt, snapshot_id) From cd0a8217c83dbef867d71e03a0fab3763489ccfc Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Sat, 9 Nov 2024 16:52:26 -0300 Subject: [PATCH 03/12] fixed mac not recovered properly --- evidence/parse.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/evidence/parse.py b/evidence/parse.py index 3ec476b..b98f840 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -4,6 +4,7 @@ import logging from dmidecode import DMIParse from json_repair import repair_json +from evidence.parse_details import get_lshw_child from evidence.models import Annotation from evidence.xapian import index @@ -12,6 +13,23 @@ from utils.constants import CHASSIS_DH logger = logging.getLogger('django') +def get_mac(lshw): + try: + if type(lshw) is dict: + hw = lshw + else: + hw = json.loads(lshw) + except json.decoder.JSONDecodeError: + hw = json.loads(repair_json(lshw)) + + networks = [] + get_lshw_child(hw, networks, 'network') + + if nets_sorted: + mac = nets_sorted[0]['serial'] + logger.debug("The snapshot has the following MAC: %s" , mac) + return mac + def get_network_cards(child, nets): if child['id'] == 'network' and "PCI:" in child.get("businfo"): From 37631244400a12bbe998ac77ca0087a3d94b8e96 Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 03:41:08 -0300 Subject: [PATCH 04/12] minor fix in mac parsing --- evidence/parse.py | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/evidence/parse.py b/evidence/parse.py index b98f840..fd68e06 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -22,8 +22,10 @@ def get_mac(lshw): except json.decoder.JSONDecodeError: hw = json.loads(repair_json(lshw)) - networks = [] - get_lshw_child(hw, networks, 'network') + nets = [] + get_lshw_child(hw, nets, 'network') + + nets_sorted = sorted(nets, key=lambda x: x['businfo']) if nets_sorted: mac = nets_sorted[0]['serial'] @@ -31,36 +33,6 @@ def get_mac(lshw): return mac -def get_network_cards(child, nets): - if child['id'] == 'network' and "PCI:" in child.get("businfo"): - nets.append(child) - if child.get('children'): - [get_network_cards(x, nets) for x in child['children']] - - -def get_mac(lshw): - nets = [] - try: - if type(lshw) is dict: - hw = lshw - else: - hw = json.loads(lshw) - except json.decoder.JSONDecodeError: - hw = json.loads(repair_json(lshw)) - - try: - get_network_cards(hw, nets) - except Exception as ss: - logger.warning("%s", ss) - return - - nets_sorted = sorted(nets, key=lambda x: x['businfo']) - # This funcion get the network card integrated in motherboard - # integrate = [x for x in nets if "pci@0000:00:" in x.get('businfo', '')] - - if nets_sorted: - return nets_sorted[0]['serial'] - class Build: def __init__(self, evidence_json, user, check=False): From 80083caf993734f503161ea59470150c89cdba42 Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 04:06:22 -0300 Subject: [PATCH 05/12] better error messages --- evidence/forms.py | 20 ++++++++++++++------ evidence/management/commands/up_snapshots.py | 6 +++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/evidence/forms.py b/evidence/forms.py index dab7a60..353bbf4 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -33,13 +33,21 @@ class UploadForm(forms.Form): exist_annotation = Annotation.objects.filter( uuid=file_json['uuid'] ).first() - + if exist_annotation: - raise ValidationError("error: {} exist".format(file_name)) - - except Exception: - raise ValidationError("error in: {}".format(file_name)) - + raise ValidationError( + _("The snapshot already exists"), + code="duplicate_snapshot", + params={"file_name": file_name}, + ) + + #Caught any error and display it as Validation Error so the Form handles it + except Exception as e: + raise ValidationError( + _("Error on '%(file_name)s': %(error)s"), + code="error", + params={"file_name": file_name, "error": getattr(e, 'message', str(e))}, + ) self.evidences.append((file_name, file_json)) return True diff --git a/evidence/management/commands/up_snapshots.py b/evidence/management/commands/up_snapshots.py index 5f699a7..760fab3 100644 --- a/evidence/management/commands/up_snapshots.py +++ b/evidence/management/commands/up_snapshots.py @@ -56,14 +56,14 @@ class Command(BaseCommand): except json.JSONDecodeError as e: logger.error("JSON decode error in file %s: %s", filepath, e) - raise ValueError(f"Invalid JSON format in file {filepath}") from e + raise ValueError(f"Invalid JSON format in file. Check for file integrity.") from e except FileNotFoundError as e: logger.error("File not found: %s", filepath) - raise FileNotFoundError(f"File not found: {filepath}") from e + raise FileNotFoundError(f"File not found") from e #or we cath'em all except Exception as e: logger.exception("Unexpected error when opening file %s: %s", filepath, e) - raise Exception(f"Unexpected error when opening file {filepath}") from e + raise Exception(f"Unexpected error when opening file") from e def parsing(self): for s, p in self.snapshots: From 90f057288dd73919cf6a958921b613a835d476ae Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 04:28:41 -0300 Subject: [PATCH 06/12] upload form now displays error messages once --- evidence/templates/upload.html | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/evidence/templates/upload.html b/evidence/templates/upload.html index 056cf55..2a76b76 100644 --- a/evidence/templates/upload.html +++ b/evidence/templates/upload.html @@ -13,18 +13,18 @@ {% csrf_token %} {% if form.errors %} {% endif %} -{% bootstrap_form form %} -
- {% translate "Cancel" %} +{% bootstrap_form form alert_error_type="fields" %} +
+ {% translate "Cancel" %}
From b4c18535fb5f776b2f6a09c570aa8c50ad4f0d5c Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 14:12:26 -0300 Subject: [PATCH 07/12] grammar fix and todo added --- evidence/forms.py | 3 +-- evidence/views.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evidence/forms.py b/evidence/forms.py index 353bbf4..0ac251f 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -38,10 +38,9 @@ class UploadForm(forms.Form): raise ValidationError( _("The snapshot already exists"), code="duplicate_snapshot", - params={"file_name": file_name}, ) - #Caught any error and display it as Validation Error so the Form handles it + #Catch any error and display it as Validation Error so the Form handles it except Exception as e: raise ValidationError( _("Error on '%(file_name)s': %(error)s"), diff --git a/evidence/views.py b/evidence/views.py index 201a621..65a033b 100644 --- a/evidence/views.py +++ b/evidence/views.py @@ -51,6 +51,7 @@ class UploadView(DashboardView, FormView): return response def form_invalid(self, form): + #TODO: change file_input field class to "is-invalid" if any errors occur response = super().form_invalid(form) return response From 8dade92f7e4067d9d69728b45bc0c2d85447fc46 Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 14:47:44 -0300 Subject: [PATCH 08/12] changed form validation onto field --- evidence/forms.py | 2 +- evidence/templates/upload.html | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/evidence/forms.py b/evidence/forms.py index 0ac251f..a2c0020 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -15,7 +15,7 @@ from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): evidence_file = MultipleFileField(label=_("File")) - def clean(self): + def clean_evidence_file(self): self.evidences = [] data = self.cleaned_data.get('evidence_file') if not data: diff --git a/evidence/templates/upload.html b/evidence/templates/upload.html index 2a76b76..6337b3e 100644 --- a/evidence/templates/upload.html +++ b/evidence/templates/upload.html @@ -8,22 +8,20 @@
+ + + {% load django_bootstrap5 %}
{% csrf_token %} -{% if form.errors %} - -{% endif %} -{% bootstrap_form form alert_error_type="fields" %} -
+ +{% bootstrap_form form alert_error_type="none" error_css_class="alert alert-danger alert-icon alert-icon-border" %} +
{% translate "Cancel" %}
From ac70a4ae25357061af3abd065349937686ae782b Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 14:49:17 -0300 Subject: [PATCH 09/12] catching pandas exception --- evidence/forms.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/evidence/forms.py b/evidence/forms.py index a2c0020..83d9e6b 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -130,7 +130,15 @@ class ImportForm(forms.Form): data = self.cleaned_data["file_import"] self.file_name = data.name - df = pd.read_excel(data) + + try: + df = pd.read_excel(data) + except Exception as e: + raise ValidationError( + _("Error on '%(file_name)s': Invalid File"), + params={"file_name": self.file_name} + ) + df.fillna('', inplace=True) data_pd = df.to_dict(orient='index') From ac48a77a2f20f3edefbb2e3667bb190994f69f6e Mon Sep 17 00:00:00 2001 From: Thomas Rusiecki Date: Mon, 11 Nov 2024 14:57:20 -0300 Subject: [PATCH 10/12] added success messages --- evidence/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evidence/views.py b/evidence/views.py index 65a033b..dabcd9d 100644 --- a/evidence/views.py +++ b/evidence/views.py @@ -1,5 +1,6 @@ import json +from django.contrib import messages from urllib.parse import urlparse from django.http import HttpResponse from django.utils.translation import gettext_lazy as _ @@ -47,11 +48,11 @@ class UploadView(DashboardView, FormView): def form_valid(self, form): form.save(self.request.user) + messages.success(self.request, _("Evidence uploaded successfully.")) response = super().form_valid(form) return response def form_invalid(self, form): - #TODO: change file_input field class to "is-invalid" if any errors occur response = super().form_invalid(form) return response @@ -71,6 +72,7 @@ class ImportView(DashboardView, FormView): def form_valid(self, form): form.save() + messages.success(self.request, _("Evidence imported successfully.")) response = super().form_valid(form) return response From a748b900ca3d095dc53f59fde44dd5c02eebc225 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 12 Nov 2024 14:31:46 +0100 Subject: [PATCH 11/12] recover old up_snapshot functionality - We don't need the if settings.DEBUG anymore, we have a logger configuration that when DEBUG is there, a trace is always generated - restore old error message that is shorter and clearer than the new one --- evidence/management/commands/up_snapshots.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/evidence/management/commands/up_snapshots.py b/evidence/management/commands/up_snapshots.py index 760fab3..b1ca8c7 100644 --- a/evidence/management/commands/up_snapshots.py +++ b/evidence/management/commands/up_snapshots.py @@ -71,8 +71,6 @@ class Command(BaseCommand): self.devices.append(Build(s, self.user)) move_json(p, self.user.institution.name) except Exception as err: - if settings.DEBUG: - logger.exception("%s", err) snapshot_id = s.get("uuid", "") - txt = "It is not possible to parse snapshot: %s" + txt = "Could not parse snapshot: %s" logger.error(txt, snapshot_id) From 65788b36afd3243266c8de4d251adf2621478b04 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 12 Nov 2024 14:37:55 +0100 Subject: [PATCH 12/12] up_snapshots: simplify and clarify errors specially, do not halt in case of errors, just print them in logs --- evidence/management/commands/up_snapshots.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/evidence/management/commands/up_snapshots.py b/evidence/management/commands/up_snapshots.py index b1ca8c7..36f5c28 100644 --- a/evidence/management/commands/up_snapshots.py +++ b/evidence/management/commands/up_snapshots.py @@ -54,23 +54,15 @@ class Command(BaseCommand): self.snapshots.append((content, path_name)) - except json.JSONDecodeError as e: - logger.error("JSON decode error in file %s: %s", filepath, e) - raise ValueError(f"Invalid JSON format in file. Check for file integrity.") from e - except FileNotFoundError as e: - logger.error("File not found: %s", filepath) - raise FileNotFoundError(f"File not found") from e - #or we cath'em all except Exception as e: - logger.exception("Unexpected error when opening file %s: %s", filepath, e) - raise Exception(f"Unexpected error when opening file") from e + logger.error("Could not open file %s: %s", filepath, e) def parsing(self): for s, p in self.snapshots: try: self.devices.append(Build(s, self.user)) move_json(p, self.user.institution.name) - except Exception as err: + except Exception as e: snapshot_id = s.get("uuid", "") - txt = "Could not parse snapshot: %s" - logger.error(txt, snapshot_id) + txt = "Could not parse snapshot %s: %s" + logger.error(txt, snapshot_id, e)