From fcb07a73375666b34fc36db6322950bc0f4d53e1 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 25 Oct 2024 17:36:13 +0200 Subject: [PATCH] Add reindex from snapshots files --- dashboard/views.py | 2 +- evidence/management/commands/reindex.py | 86 +++++++++++++++++++++++++ evidence/models.py | 2 +- evidence/parse.py | 37 +++++++---- evidence/xapian.py | 14 ++-- utils/device.py | 16 +++++ 6 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 evidence/management/commands/reindex.py diff --git a/dashboard/views.py b/dashboard/views.py index f056523..36369c5 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -66,7 +66,7 @@ class SearchView(InventaryMixin): limit ) - if not matches.size(): + if not matches or not matches.size(): return self.search_hids(query, offset, limit) devices = [] diff --git a/evidence/management/commands/reindex.py b/evidence/management/commands/reindex.py new file mode 100644 index 0000000..b43b4b9 --- /dev/null +++ b/evidence/management/commands/reindex.py @@ -0,0 +1,86 @@ +import os +import json +import logging + +from django.core.management.base import BaseCommand +from django.conf import settings + +from utils.device import create_annotation, create_doc, create_index +from user.models import Institution +from evidence.parse import Build + + +logger = logging.getLogger('django') + + +class Command(BaseCommand): + help = "Reindex snapshots" + snapshots = [] + EVIDENCES = settings.EVIDENCES_DIR + + def handle(self, *args, **kwargs): + if os.path.isdir(self.EVIDENCES): + self.read_files(self.EVIDENCES) + + self.parsing() + + def read_files(self, directory): + for filename in os.listdir(directory): + filepath = os.path.join(directory, filename) + if not os.path.isdir(filepath): + continue + + institution = Institution.objects.filter(name=filename).first() + + if not institution: + continue + user = institution.user_set.filter(is_admin=True).first() + if not user: + txt = "Error No there are Admins for the institution: {}".format( + institution.name + ) + logger.exception(txt) + continue + + snapshots_path = os.path.join(filepath, "snapshots") + placeholders_path = os.path.join(filepath, "placeholders") + + for f in os.listdir(snapshots_path): + f_path = os.path.join(snapshots_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + for f in os.listdir(placeholders_path): + f_path = os.path.join(placeholders_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + def open(self, filepath, user): + with open(filepath, 'r') as file: + content = json.loads(file.read()) + self.snapshots.append((content, user, filepath)) + + def parsing(self): + for s, user, f_path in self.snapshots: + if s.get("type") == "Websnapshot": + self.build_placeholder(s, user, f_path) + else: + self.build_snapshot(s, user, f_path) + + def build_placeholder(self, s, user, f_path): + try: + create_index(s, user) + create_annotation(s, user, commit=True) + except Exception as err: + txt = "Error: in placeholder {} \n{}".format(f_path, err) + logger.exception(txt) + + def build_snapshot(self, s, user, f_path): + try: + Build(s, user) + except Exception as err: + txt = "Error: in Snapshot {} \n{}".format(f_path, err) + logger.exception(txt) + diff --git a/evidence/models.py b/evidence/models.py index 1f31406..8c5d6f9 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -61,7 +61,7 @@ class Evidence: self.get_owner() qry = 'uuid:"{}"'.format(self.uuid) matches = search(self.owner, qry, limit=1) - if matches.size() < 0: + if matches and matches.size() < 0: return for xa in matches: diff --git a/evidence/parse.py b/evidence/parse.py index 14f3c1c..944dbd3 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -1,15 +1,16 @@ -import os import json -import shutil import hashlib +import logging -from datetime import datetime from dmidecode import DMIParse from json_repair import repair_json from evidence.models import Annotation from evidence.xapian import index -from utils.constants import ALGOS, CHASSIS_DH +from utils.constants import CHASSIS_DH + + +logger = logging.getLogger('django') def get_network_cards(child, nets): @@ -17,15 +18,18 @@ def get_network_cards(child, nets): nets.append(child) if child.get('children'): [get_network_cards(x, nets) for x in child['children']] - - + + def get_mac(lshw): nets = [] try: - hw = json.loads(lshw) + if type(lshw) is dict: + hw = lshw + else: + hw = json.loads(lshw) except json.decoder.JSONDecodeError: hw = json.loads(repair_json(lshw)) - + try: get_network_cards(hw, nets) except Exception as ss: @@ -74,10 +78,21 @@ class Build: serial_number = device.get("serialNumber", '') sku = device.get("sku", '') hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + + return hashlib.sha3_256(hid.encode()).hexdigest() def create_annotations(self): + annotation = Annotation.objects.filter( + uuid=self.uuid, + owner=self.user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid) + logger.exception(txt) + return for k, v in self.algorithms.items(): Annotation.objects.create( @@ -99,7 +114,7 @@ class Build: def get_sku(self): return self.dmi.get("System")[0].get("SKU Number", "n/a").strip() - + def get_chassis(self): return self.dmi.get("Chassis")[0].get("Type", '_virtual') @@ -115,7 +130,7 @@ class Build: if not snapshot["data"].get('lshw'): return f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + lshw = snapshot["data"]["lshw"] # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" diff --git a/evidence/xapian.py b/evidence/xapian.py index 3c0361c..98da706 100644 --- a/evidence/xapian.py +++ b/evidence/xapian.py @@ -11,7 +11,10 @@ import xapian def search(institution, qs, offset=0, limit=10): - database = xapian.Database("db") + try: + database = xapian.Database("db") + except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): + return qp = xapian.QueryParser() qp.set_database(database) @@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10): def index(institution, uuid, snap): uuid = 'uuid:"{}"'.format(uuid) - try: - matches = search(institution, uuid, limit=1) - if matches.size() > 0: - return - except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): - pass + matches = search(institution, uuid, limit=1) + if matches and matches.size() > 0: + return database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN) indexer = xapian.TermGenerator() diff --git a/utils/device.py b/utils/device.py index a358266..36c47e5 100644 --- a/utils/device.py +++ b/utils/device.py @@ -2,12 +2,17 @@ import json import uuid import hashlib import datetime +import logging from django.core.exceptions import ValidationError from evidence.xapian import index from evidence.models import Annotation from device.models import Device + +logger = logging.getLogger('django') + + def create_doc(data): if not data: return @@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False): 'value': doc['CUSTOMER_ID'], } if commit: + annotation = Annotation.objects.filter( + uuid=doc["uuid"], + owner=user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"]) + logger.exception(txt) + return annotation + return Annotation.objects.create(**data) return Annotation(**data)