diff --git a/api/urls.py b/api/urls.py index 5eac279..4fa86d6 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,9 +6,11 @@ from django.urls import path app_name = 'api' urlpatterns = [ - path('snapshot/', views.NewSnapshot, name='new_snapshot'), - path('tokens/', views.TokenView.as_view(), name='tokens'), - path('tokens/new', views.TokenNewView.as_view(), name='new_token'), - path("tokens//edit", views.EditTokenView.as_view(), name="edit_token"), - path('tokens//del', views.TokenDeleteView.as_view(), name='delete_token'), + path('v1/snapshot/', views.NewSnapshotView.as_view(), name='new_snapshot'), + path('v1/annotation//', views.AddAnnotationView.as_view(), name='new_annotation'), + path('v1/device//', views.DetailsDeviceView.as_view(), name='device'), + path('v1/tokens/', views.TokenView.as_view(), name='tokens'), + path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'), + path("v1/tokens//edit", views.EditTokenView.as_view(), name="edit_token"), + path('v1/tokens//del', views.TokenDeleteView.as_view(), name='delete_token'), ] diff --git a/api/views.py b/api/views.py index 69da2a2..676df18 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,6 @@ import json +import uuid +import logging from uuid import uuid4 @@ -7,6 +9,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator from django_tables2 import SingleTableView from django.views.generic.edit import ( CreateView, @@ -15,81 +18,118 @@ from django.views.generic.edit import ( ) from utils.save_snapshots import move_json, save_in_disk +from django.views.generic.edit import View from dashboard.mixins import DashboardView from evidence.models import Annotation +from evidence.parse_details import ParseSnapshot from evidence.parse import Build +from device.models import Device from api.models import Token from api.tables import TokensTable -@csrf_exempt -def NewSnapshot(request): - # Accept only posts - if request.method != 'POST': - return JsonResponse({'error': 'Invalid request method'}, status=400) - - # Authentication - auth_header = request.headers.get('Authorization') - if not auth_header or not auth_header.startswith('Bearer '): - return JsonResponse({'error': 'Invalid or missing token'}, status=401) - - token = auth_header.split(' ')[1] - tk = Token.objects.filter(token=token).first() - if not tk: - return JsonResponse({'error': 'Invalid or missing token'}, status=401) - - # Validation snapshot - try: - data = json.loads(request.body) - except json.JSONDecodeError: - return JsonResponse({'error': 'Invalid JSON'}, status=400) - - # try: - # Build(data, None, check=True) - # except Exception: - # return JsonResponse({'error': 'Invalid Snapshot'}, status=400) - - exist_annotation = Annotation.objects.filter( - uuid=data['uuid'] - ).first() - - if exist_annotation: - txt = "error: the snapshot {} exist".format(data['uuid']) - return JsonResponse({'status': txt}, status=500) - - # Process snapshot - path_name = save_in_disk(data, tk.owner.institution.name) - - try: - Build(data, tk.owner) - except Exception as err: - return JsonResponse({'status': f"fail: {err}"}, status=500) - - annotation = Annotation.objects.filter( - uuid=data['uuid'], - type=Annotation.Type.SYSTEM, - # TODO this is hardcoded, it should select the user preferred algorithm - key="hidalgo1", - owner=tk.owner.institution - ).first() +logger = logging.getLogger('django') - if not annotation: - return JsonResponse({'status': 'fail'}, status=500) +class ApiMixing(View): - url_args = reverse_lazy("device:details", args=(annotation.value,)) - url = request.build_absolute_uri(url_args) + @method_decorator(csrf_exempt) + def dispatch(self, *args, **kwargs): + return super().dispatch(*args, **kwargs) - response = { - "status": "success", - "dhid": annotation.value[:6].upper(), - "url": url, - # TODO replace with public_url when available - "public_url": url - } - move_json(path_name, tk.owner.institution.name) + def auth(self): + # Authentication + auth_header = self.request.headers.get('Authorization') + if not auth_header or not auth_header.startswith('Bearer '): + logger.exception("Invalid or missing token {}".format(auth_header)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) - return JsonResponse(response, status=200) + token = auth_header.split(' ')[1].strip("'").strip('"') + try: + uuid.UUID(token) + except Exception: + logger.exception("Invalid token {}".format(token)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) + + self.tk = Token.objects.filter(token=token).first() + + if not self.tk: + logger.exception("Invalid or missing token {}".format(token)) + return JsonResponse({'error': 'Invalid or missing token'}, status=401) + + +class NewSnapshotView(ApiMixing): + + def get(self, request, *args, **kwargs): + return JsonResponse({}, status=404) + + def post(self, request, *args, **kwargs): + response = self.auth() + if response: + return response + + # Validation snapshot + try: + data = json.loads(request.body) + except json.JSONDecodeError: + logger.exception("Invalid Snapshot of user {}".format(self.tk.owner)) + return JsonResponse({'error': 'Invalid JSON'}, status=500) + + # Process snapshot + path_name = save_in_disk(data, self.tk.owner.institution.name) + + # try: + # Build(data, None, check=True) + # except Exception: + # return JsonResponse({'error': 'Invalid Snapshot'}, status=400) + + if not data.get("uuid"): + txt = "error: the snapshot not have uuid" + logger.exception(txt) + return JsonResponse({'status': txt}, status=500) + + exist_annotation = Annotation.objects.filter( + uuid=data['uuid'] + ).first() + + if exist_annotation: + txt = "error: the snapshot {} exist".format(data['uuid']) + logger.exception(txt) + return JsonResponse({'status': txt}, status=500) + + + try: + Build(data, self.tk.owner) + except Exception as err: + logger.exception(err) + return JsonResponse({'status': f"fail: {err}"}, status=500) + + annotation = Annotation.objects.filter( + uuid=data['uuid'], + type=Annotation.Type.SYSTEM, + # TODO this is hardcoded, it should select the user preferred algorithm + key="hidalgo1", + owner=self.tk.owner.institution + ).first() + + + if not annotation: + logger.exception("Error: No annotation for uuid: {}".format(data["uuid"])) + return JsonResponse({'status': 'fail'}, status=500) + + url_args = reverse_lazy("device:details", args=(annotation.value,)) + url = request.build_absolute_uri(url_args) + + response = { + "status": "success", + "dhid": annotation.value[:6].upper(), + "url": url, + # TODO replace with public_url when available + "public_url": url + } + move_json(path_name, self.tk.owner.institution.name) + + return JsonResponse(response, status=200) class TokenView(DashboardView, SingleTableView): @@ -165,3 +205,99 @@ class EditTokenView(DashboardView, UpdateView): ) kwargs = super().get_form_kwargs() return kwargs + + +class DetailsDeviceView(ApiMixing): + + def get(self, request, *args, **kwargs): + response = self.auth() + if response: + return response + + self.pk = kwargs['pk'] + self.object = Device(id=self.pk) + + if not self.object.last_evidence: + return JsonResponse({}, status=404) + + if self.object.owner != self.tk.owner.institution: + return JsonResponse({}, status=403) + + data = self.get_data() + return JsonResponse(data, status=200) + + def post(self, request, *args, **kwargs): + return JsonResponse({}, status=404) + + def get_data(self): + data = {} + self.object.initial() + self.object.get_last_evidence() + evidence = self.object.last_evidence + + if evidence.is_legacy(): + data.update({ + "device": evidence.get("device"), + "components": evidence.get("components"), + }) + else: + evidence.get_doc() + snapshot = ParseSnapshot(evidence.doc).snapshot_json + data.update({ + "device": snapshot.get("device"), + "components": snapshot.get("components"), + }) + + uuids = Annotation.objects.filter( + owner=self.tk.owner.institution, + value=self.pk + ).values("uuid") + + annotations = Annotation.objects.filter( + uuid__in=uuids, + owner=self.tk.owner.institution, + type = Annotation.Type.USER + ).values_list("key", "value") + + data.update({"annotations": list(annotations)}) + return data + + +class AddAnnotationView(ApiMixing): + + def post(self, request, *args, **kwargs): + response = self.auth() + if response: + return response + + self.pk = kwargs['pk'] + institution = self.tk.owner.institution + self.annotation = Annotation.objects.filter( + owner=institution, + value=self.pk, + type=Annotation.Type.SYSTEM + ).first() + + if not self.annotation: + return JsonResponse({}, status=404) + + try: + data = json.loads(request.body) + key = data["key"] + value = data["value"] + except Exception: + logger.exception("Invalid Snapshot of user {}".format(self.tk.owner)) + return JsonResponse({'error': 'Invalid JSON'}, status=500) + + Annotation.objects.create( + uuid=self.annotation.uuid, + owner=self.tk.owner.institution, + type = Annotation.Type.USER, + key = key, + value = value + ) + + return JsonResponse({"status": "success"}, status=200) + + def get(self, request, *args, **kwargs): + return JsonResponse({}, status=404) diff --git a/dashboard/views.py b/dashboard/views.py index f056523..36369c5 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -66,7 +66,7 @@ class SearchView(InventaryMixin): limit ) - if not matches.size(): + if not matches or not matches.size(): return self.search_hids(query, offset, limit) devices = [] diff --git a/device/models.py b/device/models.py index 1f43476..da53de8 100644 --- a/device/models.py +++ b/device/models.py @@ -1,8 +1,7 @@ from django.db import models, connection -from utils.constants import STR_SM_SIZE, STR_SIZE, STR_EXTEND_SIZE, ALGOS +from utils.constants import ALGOS from evidence.models import Annotation, Evidence -from user.models import User from lot.models import DeviceLot diff --git a/evidence/management/commands/reindex.py b/evidence/management/commands/reindex.py new file mode 100644 index 0000000..b43b4b9 --- /dev/null +++ b/evidence/management/commands/reindex.py @@ -0,0 +1,86 @@ +import os +import json +import logging + +from django.core.management.base import BaseCommand +from django.conf import settings + +from utils.device import create_annotation, create_doc, create_index +from user.models import Institution +from evidence.parse import Build + + +logger = logging.getLogger('django') + + +class Command(BaseCommand): + help = "Reindex snapshots" + snapshots = [] + EVIDENCES = settings.EVIDENCES_DIR + + def handle(self, *args, **kwargs): + if os.path.isdir(self.EVIDENCES): + self.read_files(self.EVIDENCES) + + self.parsing() + + def read_files(self, directory): + for filename in os.listdir(directory): + filepath = os.path.join(directory, filename) + if not os.path.isdir(filepath): + continue + + institution = Institution.objects.filter(name=filename).first() + + if not institution: + continue + user = institution.user_set.filter(is_admin=True).first() + if not user: + txt = "Error No there are Admins for the institution: {}".format( + institution.name + ) + logger.exception(txt) + continue + + snapshots_path = os.path.join(filepath, "snapshots") + placeholders_path = os.path.join(filepath, "placeholders") + + for f in os.listdir(snapshots_path): + f_path = os.path.join(snapshots_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + for f in os.listdir(placeholders_path): + f_path = os.path.join(placeholders_path, f) + + if f_path[-5:] == ".json" and os.path.isfile(f_path): + self.open(f_path, user) + + def open(self, filepath, user): + with open(filepath, 'r') as file: + content = json.loads(file.read()) + self.snapshots.append((content, user, filepath)) + + def parsing(self): + for s, user, f_path in self.snapshots: + if s.get("type") == "Websnapshot": + self.build_placeholder(s, user, f_path) + else: + self.build_snapshot(s, user, f_path) + + def build_placeholder(self, s, user, f_path): + try: + create_index(s, user) + create_annotation(s, user, commit=True) + except Exception as err: + txt = "Error: in placeholder {} \n{}".format(f_path, err) + logger.exception(txt) + + def build_snapshot(self, s, user, f_path): + try: + Build(s, user) + except Exception as err: + txt = "Error: in Snapshot {} \n{}".format(f_path, err) + logger.exception(txt) + diff --git a/evidence/models.py b/evidence/models.py index 35f97aa..8c5d6f9 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -3,7 +3,7 @@ import json from dmidecode import DMIParse from django.db import models -from utils.constants import STR_SM_SIZE, STR_EXTEND_SIZE, CHASSIS_DH +from utils.constants import STR_EXTEND_SIZE, CHASSIS_DH from evidence.xapian import search from evidence.parse_details import ParseSnapshot from user.models import User, Institution @@ -61,13 +61,13 @@ class Evidence: self.get_owner() qry = 'uuid:"{}"'.format(self.uuid) matches = search(self.owner, qry, limit=1) - if matches.size() < 0: + if matches and matches.size() < 0: return for xa in matches: self.doc = json.loads(xa.document.get_data()) - if self.doc.get("software") == "workbench-script": + if not self.is_legacy(): dmidecode_raw = self.doc["data"]["dmidecode"] self.dmi = DMIParse(dmidecode_raw) @@ -80,7 +80,7 @@ class Evidence: self.created = self.annotations.last().created def get_components(self): - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc.get('components', []) self.set_components() return self.components @@ -92,7 +92,7 @@ class Evidence: return "" return list(self.doc.get('kv').values())[0] - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['manufacturer'] return self.dmi.manufacturer().strip() @@ -104,13 +104,13 @@ class Evidence: return "" return list(self.doc.get('kv').values())[1] - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['model'] return self.dmi.model().strip() def get_chassis(self): - if self.doc.get("software") != "workbench-script": + if self.is_legacy(): return self.doc['device']['model'] chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') @@ -132,3 +132,6 @@ class Evidence: def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] + + def is_legacy(self): + return self.doc.get("software") != "workbench-script" diff --git a/evidence/parse.py b/evidence/parse.py index 14f3c1c..944dbd3 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -1,15 +1,16 @@ -import os import json -import shutil import hashlib +import logging -from datetime import datetime from dmidecode import DMIParse from json_repair import repair_json from evidence.models import Annotation from evidence.xapian import index -from utils.constants import ALGOS, CHASSIS_DH +from utils.constants import CHASSIS_DH + + +logger = logging.getLogger('django') def get_network_cards(child, nets): @@ -17,15 +18,18 @@ def get_network_cards(child, nets): nets.append(child) if child.get('children'): [get_network_cards(x, nets) for x in child['children']] - - + + def get_mac(lshw): nets = [] try: - hw = json.loads(lshw) + if type(lshw) is dict: + hw = lshw + else: + hw = json.loads(lshw) except json.decoder.JSONDecodeError: hw = json.loads(repair_json(lshw)) - + try: get_network_cards(hw, nets) except Exception as ss: @@ -74,10 +78,21 @@ class Build: serial_number = device.get("serialNumber", '') sku = device.get("sku", '') hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + + return hashlib.sha3_256(hid.encode()).hexdigest() def create_annotations(self): + annotation = Annotation.objects.filter( + uuid=self.uuid, + owner=self.user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(self.uuid) + logger.exception(txt) + return for k, v in self.algorithms.items(): Annotation.objects.create( @@ -99,7 +114,7 @@ class Build: def get_sku(self): return self.dmi.get("System")[0].get("SKU Number", "n/a").strip() - + def get_chassis(self): return self.dmi.get("Chassis")[0].get("Type", '_virtual') @@ -115,7 +130,7 @@ class Build: if not snapshot["data"].get('lshw'): return f"{manufacturer}{model}{chassis}{serial_number}{sku}" - + lshw = snapshot["data"]["lshw"] # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" diff --git a/evidence/xapian.py b/evidence/xapian.py index 3c0361c..98da706 100644 --- a/evidence/xapian.py +++ b/evidence/xapian.py @@ -11,7 +11,10 @@ import xapian def search(institution, qs, offset=0, limit=10): - database = xapian.Database("db") + try: + database = xapian.Database("db") + except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): + return qp = xapian.QueryParser() qp.set_database(database) @@ -31,12 +34,9 @@ def search(institution, qs, offset=0, limit=10): def index(institution, uuid, snap): uuid = 'uuid:"{}"'.format(uuid) - try: - matches = search(institution, uuid, limit=1) - if matches.size() > 0: - return - except (xapian.DatabaseNotFoundError, xapian.DatabaseOpeningError): - pass + matches = search(institution, uuid, limit=1) + if matches and matches.size() > 0: + return database = xapian.WritableDatabase("db", xapian.DB_CREATE_OR_OPEN) indexer = xapian.TermGenerator() diff --git a/utils/device.py b/utils/device.py index a358266..36c47e5 100644 --- a/utils/device.py +++ b/utils/device.py @@ -2,12 +2,17 @@ import json import uuid import hashlib import datetime +import logging from django.core.exceptions import ValidationError from evidence.xapian import index from evidence.models import Annotation from device.models import Device + +logger = logging.getLogger('django') + + def create_doc(data): if not data: return @@ -76,6 +81,17 @@ def create_annotation(doc, user, commit=False): 'value': doc['CUSTOMER_ID'], } if commit: + annotation = Annotation.objects.filter( + uuid=doc["uuid"], + owner=user.institution, + type=Annotation.Type.SYSTEM, + ) + + if annotation: + txt = "Warning: Snapshot {} exist as annotation !!".format(doc["uuid"]) + logger.exception(txt) + return annotation + return Annotation.objects.create(**data) return Annotation(**data)