Add reindex from snapshots files

dashboard/views.py

@@ -66,7 +66,7 @@ class SearchView(InventaryMixin):
             limit
         )
 
-        if not matches.size():
+        if not matches or not matches.size():
             return self.search_hids(query, offset, limit)
 
         devices = []

evidence/management/commands/reindex.py

@@ -0,0 +1,86 @@
+import os
+import json
+import logging
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from utils.device import create_annotation, create_doc, create_index
+from user.models import Institution
+from evidence.parse import Build
+
+
+logger = logging.getLogger('django')
+
+
+class Command(BaseCommand):
+    help = "Reindex snapshots"
+    snapshots = []
+    EVIDENCES = settings.EVIDENCES_DIR
+
+    def handle(self, *args, **kwargs):
+        if os.path.isdir(self.EVIDENCES):
+            self.read_files(self.EVIDENCES)
+
+        self.parsing()
+
+    def read_files(self, directory):
+        for filename in os.listdir(directory):
+            filepath = os.path.join(directory, filename)
+            if not os.path.isdir(filepath):
+                continue
+
+            institution = Institution.objects.filter(name=filename).first()
+
+            if not institution:
+                continue
+            user = institution.user_set.filter(is_admin=True).first()
+            if not user:
+                txt = "Error No there are Admins for the institution: {}".format(
+                    institution.name
+                )
+                logger.exception(txt)
+                continue
+
+            snapshots_path = os.path.join(filepath, "snapshots")
+            placeholders_path = os.path.join(filepath, "placeholders")
+
+            for f in os.listdir(snapshots_path):
+                f_path = os.path.join(snapshots_path, f)
+
+                if f_path[-5:] == ".json" and os.path.isfile(f_path):
+                    self.open(f_path, user)
+
+            for f in os.listdir(placeholders_path):
+                f_path = os.path.join(placeholders_path, f)
+
+                if f_path[-5:] == ".json" and os.path.isfile(f_path):
+                    self.open(f_path, user)
+
+    def open(self, filepath, user):
+        with open(filepath, 'r') as file:
+            content = json.loads(file.read())
+            self.snapshots.append((content, user, filepath))
+
+    def parsing(self):
+        for s, user, f_path in self.snapshots:
+            if s.get("type") == "Websnapshot":
+                self.build_placeholder(s, user, f_path)
+            else:
+                self.build_snapshot(s, user, f_path)
+
+    def build_placeholder(self, s, user, f_path):
+            try:
+                create_index(s, user)
+                create_annotation(s, user, commit=True)
+            except Exception as err:
+                txt = "Error: in placeholder {} \n{}".format(f_path, err)
+                logger.exception(txt)
+                
+    def build_snapshot(self, s, user, f_path):
+            try:
+                Build(s, user)
+            except Exception as err:
+                txt = "Error: in Snapshot {} \n{}".format(f_path, err)
+                logger.exception(txt)
+

evidence/models.py

@@ -61,7 +61,7 @@ class Evidence:
             self.get_owner()
         qry = 'uuid:"{}"'.format(self.uuid)
         matches = search(self.owner, qry, limit=1)
-        if matches.size() < 0:
+        if matches and matches.size() < 0:
             return
 
         for xa in matches:

evidence/parse.py

@@ -1,15 +1,16 @@
-import os
 import json
-import shutil
 import hashlib
+import logging
 
-from datetime import datetime
 from dmidecode import DMIParse
 from json_repair import repair_json
 
 from evidence.models import Annotation
 from evidence.xapian import index
-from utils.constants import ALGOS, CHASSIS_DH
+from utils.constants import CHASSIS_DH
+
+
+logger = logging.getLogger('django')
 
 
 def get_network_cards(child, nets):
@@ -17,15 +18,18 @@ def get_network_cards(child, nets):
         nets.append(child)
     if child.get('children'):
         [get_network_cards(x, nets) for x in child['children']]
-        
-        
+
+
 def get_mac(lshw):
     nets = []
     try:
-        hw = json.loads(lshw)
+        if type(lshw) is dict:
+            hw = lshw
+        else:
+            hw = json.loads(lshw)
     except json.decoder.JSONDecodeError:
         hw = json.loads(repair_json(lshw))
-        
+
     try:
         get_network_cards(hw, nets)
     except Exception as ss:
@@ -74,10 +78,21 @@ class Build:
             serial_number = device.get("serialNumber", '')
             sku = device.get("sku", '')
             hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}"
-            
+
+
         return hashlib.sha3_256(hid.encode()).hexdigest()
 
     def create_annotations(self):
+        annotation = Annotation.objects.filter(
+                uuid=self.uuid,
+                owner=self.user.institution,
+                type=Annotation.Type.SYSTEM,
+        )
+
+        if annotation:
+            txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid)
+            logger.exception(txt)
+            return
 
         for k, v in self.algorithms.items():
             Annotation.objects.create(
@@ -99,7 +114,7 @@ class Build:
 
     def get_sku(self):
         return self.dmi.get("System")[0].get("SKU Number", "n/a").strip()
-    
+
     def get_chassis(self):
         return self.dmi.get("Chassis")[0].get("Type", '_virtual')
 
@@ -115,7 +130,7 @@ class Build:
 
         if not snapshot["data"].get('lshw'):
             return f"{manufacturer}{model}{chassis}{serial_number}{sku}"
-        
+
         lshw = snapshot["data"]["lshw"]
         # mac = get_mac2(hwinfo_raw) or ""
         mac = get_mac(lshw) or ""

evidence/xapian.py

@@ -11,7 +11,10 @@ import xapian
 
 
 def search(institution, qs, offset=0, limit=10):
-    database = xapian.Database("db")
+    try:
+        database = xapian.Database("db")
+    except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
+        return
 
     qp = xapian.QueryParser()
     qp.set_database(database)
@@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10):
 
 def index(institution, uuid, snap):
     uuid = 'uuid:"{}"'.format(uuid)
-    try:
-        matches = search(institution, uuid, limit=1)
-        if matches.size() > 0:
-            return
-    except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError):
-        pass
+    matches = search(institution, uuid, limit=1)
+    if matches and matches.size() > 0:
+        return
 
     database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN)
     indexer = xapian.TermGenerator()

utils/device.py

@@ -2,12 +2,17 @@ import json
 import uuid
 import hashlib
 import datetime
+import logging
 
 from django.core.exceptions import ValidationError
 from evidence.xapian import index
 from evidence.models import Annotation
 from device.models import Device
 
+
+logger = logging.getLogger('django')
+
+
 def create_doc(data):
     if not data:
         return
@@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False):
         'value': doc['CUSTOMER_ID'],
     }
     if commit:
+        annotation = Annotation.objects.filter(
+                uuid=doc["uuid"],
+                owner=user.institution,
+                type=Annotation.Type.SYSTEM,
+        )
+
+        if annotation:
+            txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"])
+            logger.exception(txt)
+            return annotation
+
         return Annotation.objects.create(**data)
 
     return Annotation(**data)