From 9c78ff6c1b2dbace474db91c9edd9cdd93098168 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 22 Oct 2024 10:00:34 +0200 Subject: [PATCH 1/9] add v1 in url of api --- api/urls.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/urls.py b/api/urls.py index 5eac279..17202c8 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,9 +6,9 @@ from django.urls import path app_name = 'api' urlpatterns = [ - path('snapshot/', views.NewSnapshot, name='new_snapshot'), - path('tokens/', views.TokenView.as_view(), name='tokens'), - path('tokens/new', views.TokenNewView.as_view(), name='new_token'), - path("tokens//edit", views.EditTokenView.as_view(), name="edit_token"), - path('tokens//del', views.TokenDeleteView.as_view(), name='delete_token'), + path('v1/snapshot/', views.NewSnapshot, name='new_snapshot'), + path('v1/tokens/', views.TokenView.as_view(), name='tokens'), + path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'), + path("v1/tokens//edit", views.EditTokenView.as_view(), name="edit_token"), + path('v1/tokens//del', views.TokenDeleteView.as_view(), name='delete_token'), ] From ff92e28b9711765457bb072954a5a9a6eb2196bd Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 22 Oct 2024 10:01:29 +0200 Subject: [PATCH 2/9] remove quotas in string of token --- api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 69da2a2..7ea8a76 100644 --- a/api/views.py +++ b/api/views.py @@ -33,7 +33,7 @@ def NewSnapshot(request): if not auth_header or not auth_header.startswith('Bearer '): return JsonResponse({'error': 'Invalid or missing token'}, status=401) - token = auth_header.split(' ')[1] + token = auth_header.split(' ')[1].strip("'") tk = Token.objects.filter(token=token).first() if not tk: return JsonResponse({'error': 'Invalid or missing token'}, status=401) From ba94a01062380b9f122c26fbd0612cebf91f5915 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 22 Oct 2024 10:09:53 +0200 Subject: [PATCH 3/9] remove quotas in string of token fixed more --- api/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 7ea8a76..d262da4 100644 --- a/api/views.py +++ b/api/views.py @@ -32,8 +32,9 @@ def NewSnapshot(request): auth_header = request.headers.get('Authorization') if not auth_header or not auth_header.startswith('Bearer '): return JsonResponse({'error': 'Invalid or missing token'}, status=401) + + token = auth_header.split(' ')[1].strip("'").strip('"') - token = auth_header.split(' ')[1].strip("'") tk = Token.objects.filter(token=token).first() if not tk: return JsonResponse({'error': 'Invalid or missing token'}, status=401) From 0f8815eb432583123ebef1043f892d6bfa801ca8 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 22 Oct 2024 10:57:06 +0200 Subject: [PATCH 4/9] more logs for errors 500 in server side --- api/views.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index d262da4..c2c91f1 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,6 @@ import json +import uuid +import logging from uuid import uuid4 @@ -22,6 +24,9 @@ from api.models import Token from api.tables import TokensTable +logger = logging.getLogger('django') + + @csrf_exempt def NewSnapshot(request): # Accept only posts @@ -31,19 +36,28 @@ def NewSnapshot(request): # Authentication auth_header = request.headers.get('Authorization') if not auth_header or not auth_header.startswith('Bearer '): + logger.exception("Invalid or missing token {}".format(auth_header)) return JsonResponse({'error': 'Invalid or missing token'}, status=401) token = auth_header.split(' ')[1].strip("'").strip('"') - + try: + uuid.UUID(token) + except Exception: + logger.exception("Invalid token {}".format(token)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) + tk = Token.objects.filter(token=token).first() + if not tk: + logger.exception("Invalid or missing token {}".format(token)) return JsonResponse({'error': 'Invalid or missing token'}, status=401) # Validation snapshot try: data = json.loads(request.body) except json.JSONDecodeError: - return JsonResponse({'error': 'Invalid JSON'}, status=400) + logger.exception("Invalid Snapshot of user {}".format(tk.owner)) + return JsonResponse({'error': 'Invalid JSON'}, status=500) # try: # Build(data, None, check=True) @@ -56,6 +70,7 @@ def NewSnapshot(request): if exist_annotation: txt = "error: the snapshot {} exist".format(data['uuid']) + logger.exception(txt) return JsonResponse({'status': txt}, status=500) # Process snapshot @@ -64,6 +79,7 @@ def NewSnapshot(request): try: Build(data, tk.owner) except Exception as err: + logger.exception(err) return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( @@ -76,6 +92,7 @@ def NewSnapshot(request): if not annotation: + logger.exception("Error: No annotation for uuid: {}".format(data["uuid"])) return JsonResponse({'status': 'fail'}, status=500) url_args = reverse_lazy("device:details", args=(annotation.value,)) From 9b96955c30dec8a44e499a2fe6b851e75c83dc45 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 23 Oct 2024 09:23:45 +0200 Subject: [PATCH 5/9] change def for views in api views --- api/urls.py | 2 +- api/views.py | 180 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 110 insertions(+), 72 deletions(-) diff --git a/api/urls.py b/api/urls.py index 17202c8..c3cb111 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,7 +6,7 @@ from django.urls import path app_name = 'api' urlpatterns = [ - path('v1/snapshot/', views.NewSnapshot, name='new_snapshot'), + path('v1/snapshot/', views.NewSnapshotView.as_view(), name='new_snapshot'), path('v1/tokens/', views.TokenView.as_view(), name='tokens'), path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'), path("v1/tokens//edit", views.EditTokenView.as_view(), name="edit_token"), diff --git a/api/views.py b/api/views.py index c2c91f1..6bf9cfb 100644 --- a/api/views.py +++ b/api/views.py @@ -9,6 +9,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator from django_tables2 import SingleTableView from django.views.generic.edit import ( CreateView, @@ -17,6 +18,7 @@ from django.views.generic.edit import ( ) from utils.save_snapshots import move_json, save_in_disk +from django.views.generic.edit import View from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -27,87 +29,105 @@ from api.tables import TokensTable logger = logging.getLogger('django') -@csrf_exempt -def NewSnapshot(request): - # Accept only posts - if request.method != 'POST': - return JsonResponse({'error': 'Invalid request method'}, status=400) +class ApiMixing(View): - # Authentication - auth_header = request.headers.get('Authorization') - if not auth_header or not auth_header.startswith('Bearer '): - logger.exception("Invalid or missing token {}".format(auth_header)) - return JsonResponse({'error': 'Invalid or missing token'}, status=401) - - token = auth_header.split(' ')[1].strip("'").strip('"') - try: - uuid.UUID(token) - except Exception: - logger.exception("Invalid token {}".format(token)) - return JsonResponse({'error': 'Invalid or missing token'}, status=401) - - tk = Token.objects.filter(token=token).first() - - if not tk: - logger.exception("Invalid or missing token {}".format(token)) - return JsonResponse({'error': 'Invalid or missing token'}, status=401) + @method_decorator(csrf_exempt) + def dispatch(self, *args, **kwargs): + return super().dispatch(*args, **kwargs) - # Validation snapshot - try: - data = json.loads(request.body) - except json.JSONDecodeError: - logger.exception("Invalid Snapshot of user {}".format(tk.owner)) - return JsonResponse({'error': 'Invalid JSON'}, status=500) + def auth(self): + # Authentication + auth_header = self.request.headers.get('Authorization') + if not auth_header or not auth_header.startswith('Bearer '): + logger.exception("Invalid or missing token {}".format(auth_header)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) - # try: - # Build(data, None, check=True) - # except Exception: - # return JsonResponse({'error': 'Invalid Snapshot'}, status=400) + token = auth_header.split(' ')[1].strip("'").strip('"') + try: + uuid.UUID(token) + except Exception: + logger.exception("Invalid token {}".format(token)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) - exist_annotation = Annotation.objects.filter( - uuid=data['uuid'] - ).first() + self.tk = Token.objects.filter(token=token).first() - if exist_annotation: - txt = "error: the snapshot {} exist".format(data['uuid']) - logger.exception(txt) - return JsonResponse({'status': txt}, status=500) - - # Process snapshot - path_name = save_in_disk(data, tk.owner.institution.name) - - try: - Build(data, tk.owner) - except Exception as err: - logger.exception(err) - return JsonResponse({'status': f"fail: {err}"}, status=500) - - annotation = Annotation.objects.filter( - uuid=data['uuid'], - type=Annotation.Type.SYSTEM, - # TODO this is hardcoded, it should select the user preferred algorithm - key="hidalgo1", - owner=tk.owner.institution - ).first() + if not self.tk: + logger.exception("Invalid or missing token {}".format(token)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) - if not annotation: - logger.exception("Error: No annotation for uuid: {}".format(data["uuid"])) - return JsonResponse({'status': 'fail'}, status=500) +class NewSnapshotView(ApiMixing): - url_args = reverse_lazy("device:details", args=(annotation.value,)) - url = request.build_absolute_uri(url_args) + def get(self, request, *args, **kwargs): + return JsonResponse({}, status=404) - response = { - "status": "success", - "dhid": annotation.value[:6].upper(), - "url": url, - # TODO replace with public_url when available - "public_url": url - } - move_json(path_name, tk.owner.institution.name) + def post(self, request, *args, **kwargs): + response = self.auth() + if response: + return response - return JsonResponse(response, status=200) + # Validation snapshot + try: + data = json.loads(request.body) + except json.JSONDecodeError: + logger.exception("Invalid Snapshot of user {}".format(self.tk.owner)) + return JsonResponse({'error': 'Invalid JSON'}, status=500) + + # Process snapshot + path_name = save_in_disk(data, self.tk.owner.institution.name) + + # try: + # Build(data, None, check=True) + # except Exception: + # return JsonResponse({'error': 'Invalid Snapshot'}, status=400) + + if not data.get("uuid"): + txt = "error: the snapshot not have uuid" + logger.exception(txt) + return JsonResponse({'status': txt}, status=500) + + exist_annotation = Annotation.objects.filter( + uuid=data['uuid'] + ).first() + + if exist_annotation: + txt = "error: the snapshot {} exist".format(data['uuid']) + logger.exception(txt) + return JsonResponse({'status': txt}, status=500) + + + try: + Build(data, self.tk.owner) + except Exception as err: + logger.exception(err) + return JsonResponse({'status': f"fail: {err}"}, status=500) + + annotation = Annotation.objects.filter( + uuid=data['uuid'], + type=Annotation.Type.SYSTEM, + # TODO this is hardcoded, it should select the user preferred algorithm + key="hidalgo1", + owner=self.tk.owner.institution + ).first() + + + if not annotation: + logger.exception("Error: No annotation for uuid: {}".format(data["uuid"])) + return JsonResponse({'status': 'fail'}, status=500) + + url_args = reverse_lazy("device:details", args=(annotation.value,)) + url = request.build_absolute_uri(url_args) + + response = { + "status": "success", + "dhid": annotation.value[:6].upper(), + "url": url, + # TODO replace with public_url when available + "public_url": url + } + move_json(path_name, self.tk.owner.institution.name) + + return JsonResponse(response, status=200) class TokenView(DashboardView, SingleTableView): @@ -183,3 +203,21 @@ class EditTokenView(DashboardView, UpdateView): ) kwargs = super().get_form_kwargs() return kwargs + + +class DetailsComputerView(ApiMixing): + + def get(self, request, *args, **kwargs): + response = self.auth() + if response: + return response + + try: + data = json.loads(request.body) + except: + pass + + return JsonResponse({}, status=404) + + def post(self, request, *args, **kwargs): + return JsonResponse({}, status=404) From a2b54151496b2fb9bb1fe9e588a8b9ae9076d0a8 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 23 Oct 2024 13:39:16 +0200 Subject: [PATCH 6/9] response details of device with annotations from api --- api/urls.py | 1 + api/views.py | 52 +++++++++++++++++++++++++++++++++++++++------- device/models.py | 3 +-- evidence/models.py | 15 +++++++------ 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/api/urls.py b/api/urls.py index c3cb111..296a768 100644 --- a/api/urls.py +++ b/api/urls.py @@ -7,6 +7,7 @@ app_name = 'api' urlpatterns = [ path('v1/snapshot/', views.NewSnapshotView.as_view(), name='new_snapshot'), + path('v1/device//', views.DetailsDeviceView.as_view(), name='device'), path('v1/tokens/', views.TokenView.as_view(), name='tokens'), path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'), path("v1/tokens//edit", views.EditTokenView.as_view(), name="edit_token"), diff --git a/api/views.py b/api/views.py index 6bf9cfb..795d221 100644 --- a/api/views.py +++ b/api/views.py @@ -205,19 +205,57 @@ class EditTokenView(DashboardView, UpdateView): return kwargs -class DetailsComputerView(ApiMixing): +class DetailsDeviceView(ApiMixing): def get(self, request, *args, **kwargs): response = self.auth() if response: return response - try: - data = json.loads(request.body) - except: - pass - - return JsonResponse({}, status=404) + self.pk = kwargs['pk'] + self.object = Device(id=self.pk) + + if not self.object.last_evidence: + return JsonResponse({}, status=404) + + if self.object.owner != self.tk.owner.institution: + return JsonResponse({}, status=403) + + data = self.get_data() + return JsonResponse(data, status=200) def post(self, request, *args, **kwargs): return JsonResponse({}, status=404) + + def get_data(self): + data = {} + self.object.initial() + self.object.get_last_evidence() + evidence = self.object.last_evidence + + if evidence.is_legacy(): + data.update({ + "device": evidence.get("device"), + "components": evidence.get("components"), + }) + else: + evidence.get_doc() + snapshot = ParseSnapshot(evidence.doc).snapshot_json + data.update({ + "device": snapshot.get("device"), + "components": snapshot.get("components"), + }) + + uuids = Annotation.objects.filter( + owner=self.tk.owner.institution, + value=self.pk + ).values("uuid") + + annotations = Annotation.objects.filter( + uuid__in=uuids, + owner=self.tk.owner.institution, + type = Annotation.Type.USER + ).values_list("key", "value") + + data.update({"annotations": list(annotations)}) + return data diff --git a/device/models.py b/device/models.py index 1f43476..da53de8 100644 --- a/device/models.py +++ b/device/models.py @@ -1,8 +1,7 @@ from django.db import models, connection -from utils.constants import STR_SM_SIZE, STR_SIZE, STR_EXTEND_SIZE, ALGOS +from utils.constants import ALGOS from evidence.models import Annotation, Evidence -from user.models import User from lot.models import DeviceLot diff --git a/evidence/models.py b/evidence/models.py index 35f97aa..1f31406 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -3,7 +3,7 @@ import json from dmidecode import DMIParse from django.db import models -from utils.constants import STR_SM_SIZE, STR_EXTEND_SIZE, CHASSIS_DH +from utils.constants import STR_EXTEND_SIZE, CHASSIS_DH from evidence.xapian import search from evidence.parse_details import ParseSnapshot from user.models import User, Institution @@ -67,7 +67,7 @@ class Evidence: for xa in matches: self.doc = json.loads(xa.document.get_data()) - if self.doc.get("software") == "workbench-script": + if not self.is_legacy(): dmidecode_raw = self.doc["data"]["dmidecode"] self.dmi = DMIParse(dmidecode_raw) @@ -80,7 +80,7 @@ class Evidence: self.created = self.annotations.last().created def get_components(self): - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc.get('components', []) self.set_components() return self.components @@ -92,7 +92,7 @@ class Evidence: return "" return list(self.doc.get('kv').values())[0] - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['manufacturer'] return self.dmi.manufacturer().strip() @@ -104,13 +104,13 @@ class Evidence: return "" return list(self.doc.get('kv').values())[1] - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['model'] return self.dmi.model().strip() def get_chassis(self): - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['model'] chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') @@ -132,3 +132,6 @@ class Evidence: def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] + + def is_legacy(self): + return self.doc.get("software") != "workbench-script" From 8308a313e3899b81631a7bdee3bf261b624f062e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 23 Oct 2024 13:42:09 +0200 Subject: [PATCH 7/9] fix --- api/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/views.py b/api/views.py index 795d221..b659ebb 100644 --- a/api/views.py +++ b/api/views.py @@ -21,7 +21,9 @@ from utils.save_snapshots import move_json, save_in_disk from django.views.generic.edit import View from dashboard.mixins import DashboardView from evidence.models import Annotation +from evidence.parse_details import ParseSnapshot from evidence.parse import Build +from device.models import Device from api.models import Token from api.tables import TokensTable From 31ed0507187f27411f84b564491c0893245d4465 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 24 Oct 2024 11:53:37 +0200 Subject: [PATCH 8/9] endpoint for insert annotations from api --- api/urls.py | 1 + api/views.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/api/urls.py b/api/urls.py index 296a768..4fa86d6 100644 --- a/api/urls.py +++ b/api/urls.py @@ -7,6 +7,7 @@ app_name = 'api' urlpatterns = [ path('v1/snapshot/', views.NewSnapshotView.as_view(), name='new_snapshot'), + path('v1/annotation//', views.AddAnnotationView.as_view(), name='new_annotation'), path('v1/device//', views.DetailsDeviceView.as_view(), name='device'), path('v1/tokens/', views.TokenView.as_view(), name='tokens'), path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'), diff --git a/api/views.py b/api/views.py index b659ebb..676df18 100644 --- a/api/views.py +++ b/api/views.py @@ -261,3 +261,43 @@ class DetailsDeviceView(ApiMixing): data.update({"annotations": list(annotations)}) return data + + +class AddAnnotationView(ApiMixing): + + def post(self, request, *args, **kwargs): + response = self.auth() + if response: + return response + + self.pk = kwargs['pk'] + institution = self.tk.owner.institution + self.annotation = Annotation.objects.filter( + owner=institution, + value=self.pk, + type=Annotation.Type.SYSTEM + ).first() + + if not self.annotation: + return JsonResponse({}, status=404) + + try: + data = json.loads(request.body) + key = data["key"] + value = data["value"] + except Exception: + logger.exception("Invalid Snapshot of user {}".format(self.tk.owner)) + return JsonResponse({'error': 'Invalid JSON'}, status=500) + + Annotation.objects.create( + uuid=self.annotation.uuid, + owner=self.tk.owner.institution, + type = Annotation.Type.USER, + key = key, + value = value + ) + + return JsonResponse({"status": "success"}, status=200) + + def get(self, request, *args, **kwargs): + return JsonResponse({}, status=404) From fcb07a73375666b34fc36db6322950bc0f4d53e1 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 25 Oct 2024 17:36:13 +0200 Subject: [PATCH 9/9] Add reindex from snapshots files --- dashboard/views.py | 2 +- evidence/management/commands/reindex.py | 86 +++++++++++++++++++++++++ evidence/models.py | 2 +- evidence/parse.py | 37 +++++++---- evidence/xapian.py | 14 ++-- utils/device.py | 16 +++++ 6 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 evidence/management/commands/reindex.py diff --git a/dashboard/views.py b/dashboard/views.py index f056523..36369c5 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -66,7 +66,7 @@ class SearchView(InventaryMixin): limit ) - if not matches.size(): + if not matches or not matches.size(): return self.search_hids(query, offset, limit) devices = [] diff --git a/evidence/management/commands/reindex.py b/evidence/management/commands/reindex.py new file mode 100644 index 0000000..b43b4b9 --- /dev/null +++ b/evidence/management/commands/reindex.py @@ -0,0 +1,86 @@ +import os +import json +import logging + +from django.core.management.base import BaseCommand +from django.conf import settings + +from utils.device import create_annotation, create_doc, create_index +from user.models import Institution +from evidence.parse import Build + + +logger = logging.getLogger('django') + + +class Command(BaseCommand): + help = "Reindex snapshots" + snapshots = [] + EVIDENCES = settings.EVIDENCES_DIR + + def handle(self, *args, **kwargs): + if os.path.isdir(self.EVIDENCES): + self.read_files(self.EVIDENCES) + + self.parsing() + + def read_files(self, directory): + for filename in os.listdir(directory): + filepath = os.path.join(directory, filename) + if not os.path.isdir(filepath): + continue + + institution = Institution.objects.filter(name=filename).first() + + if not institution: + continue + user = institution.user_set.filter(is_admin=True).first() + if not user: + txt = "Error No there are Admins for the institution: {}".format( + institution.name + ) + logger.exception(txt) + continue + + snapshots_path = os.path.join(filepath, "snapshots") + placeholders_path = os.path.join(filepath, "placeholders") + + for f in os.listdir(snapshots_path): + f_path = os.path.join(snapshots_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + for f in os.listdir(placeholders_path): + f_path = os.path.join(placeholders_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + def open(self, filepath, user): + with open(filepath, 'r') as file: + content = json.loads(file.read()) + self.snapshots.append((content, user, filepath)) + + def parsing(self): + for s, user, f_path in self.snapshots: + if s.get("type") == "Websnapshot": + self.build_placeholder(s, user, f_path) + else: + self.build_snapshot(s, user, f_path) + + def build_placeholder(self, s, user, f_path): + try: + create_index(s, user) + create_annotation(s, user, commit=True) + except Exception as err: + txt = "Error: in placeholder {} \n{}".format(f_path, err) + logger.exception(txt) + + def build_snapshot(self, s, user, f_path): + try: + Build(s, user) + except Exception as err: + txt = "Error: in Snapshot {} \n{}".format(f_path, err) + logger.exception(txt) + diff --git a/evidence/models.py b/evidence/models.py index 1f31406..8c5d6f9 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -61,7 +61,7 @@ class Evidence: self.get_owner() qry = 'uuid:"{}"'.format(self.uuid) matches = search(self.owner, qry, limit=1) - if matches.size() < 0: + if matches and matches.size() < 0: return for xa in matches: diff --git a/evidence/parse.py b/evidence/parse.py index 14f3c1c..944dbd3 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -1,15 +1,16 @@ -import os import json -import shutil import hashlib +import logging -from datetime import datetime from dmidecode import DMIParse from json_repair import repair_json from evidence.models import Annotation from evidence.xapian import index -from utils.constants import ALGOS, CHASSIS_DH +from utils.constants import CHASSIS_DH + + +logger = logging.getLogger('django') def get_network_cards(child, nets): @@ -17,15 +18,18 @@ def get_network_cards(child, nets): nets.append(child) if child.get('children'): [get_network_cards(x, nets) for x in child['children']] - - + + def get_mac(lshw): nets = [] try: - hw = json.loads(lshw) + if type(lshw) is dict: + hw = lshw + else: + hw = json.loads(lshw) except json.decoder.JSONDecodeError: hw = json.loads(repair_json(lshw)) - + try: get_network_cards(hw, nets) except Exception as ss: @@ -74,10 +78,21 @@ class Build: serial_number = device.get("serialNumber", '') sku = device.get("sku", '') hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + + return hashlib.sha3_256(hid.encode()).hexdigest() def create_annotations(self): + annotation = Annotation.objects.filter( + uuid=self.uuid, + owner=self.user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid) + logger.exception(txt) + return for k, v in self.algorithms.items(): Annotation.objects.create( @@ -99,7 +114,7 @@ class Build: def get_sku(self): return self.dmi.get("System")[0].get("SKU Number", "n/a").strip() - + def get_chassis(self): return self.dmi.get("Chassis")[0].get("Type", '_virtual') @@ -115,7 +130,7 @@ class Build: if not snapshot["data"].get('lshw'): return f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + lshw = snapshot["data"]["lshw"] # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" diff --git a/evidence/xapian.py b/evidence/xapian.py index 3c0361c..98da706 100644 --- a/evidence/xapian.py +++ b/evidence/xapian.py @@ -11,7 +11,10 @@ import xapian def search(institution, qs, offset=0, limit=10): - database = xapian.Database("db") + try: + database = xapian.Database("db") + except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): + return qp = xapian.QueryParser() qp.set_database(database) @@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10): def index(institution, uuid, snap): uuid = 'uuid:"{}"'.format(uuid) - try: - matches = search(institution, uuid, limit=1) - if matches.size() > 0: - return - except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): - pass + matches = search(institution, uuid, limit=1) + if matches and matches.size() > 0: + return database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN) indexer = xapian.TermGenerator() diff --git a/utils/device.py b/utils/device.py index a358266..36c47e5 100644 --- a/utils/device.py +++ b/utils/device.py @@ -2,12 +2,17 @@ import json import uuid import hashlib import datetime +import logging from django.core.exceptions import ValidationError from evidence.xapian import index from evidence.models import Annotation from device.models import Device + +logger = logging.getLogger('django') + + def create_doc(data): if not data: return @@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False): 'value': doc['CUSTOMER_ID'], } if commit: + annotation = Annotation.objects.filter( + uuid=doc["uuid"], + owner=user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"]) + logger.exception(txt) + return annotation + return Annotation.objects.create(**data) return Annotation(**data)