Import device from a file

dashboard/templates/base.html

@@ -132,7 +132,7 @@
                 </a>
                 <ul class="flex-column mb-2 ul_sidebar accordion-collapse {% if section == 'People' %}expanded{% else %}collapse{% endif %}" id="ul_placeholders" data-bs-parent="#sidebarMenu">
                   <li class="nav-item">
-                    <a class="nav-link{% if path == 'admin_people_list' %} active2{% endif %}" href="{# url 'idhub:admin_people_list' #}">
+                    <a class="nav-link{% if path == 'admin_people_list' %} active2{% endif %}" href="{% url 'evidence:import' %}">
                       {% trans 'Upload Spreadsheet' %}
                     </a>
                   </li>

device/forms.py

@@ -5,7 +5,7 @@ import datetime
 
 from django import forms
 from evidence.models import Annotation
-from evidence.xapian import search, index
+from evidence.xapian import index
 
 DEVICE_TYPES = [
     ("Desktop", "Desktop"),
@@ -62,8 +62,6 @@ class BaseDeviceFormSet(forms.BaseFormSet):
 
         if not device:
             return
-        device["manufacturer"] = ""
-        device["model"] = tag
 
         doc["device"] = device
 
@@ -71,10 +69,12 @@ class BaseDeviceFormSet(forms.BaseFormSet):
             doc["kv"] = kv
 
         date = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        
         if doc:
             doc["uuid"] = self.uuid
             doc["endTime"] = date
             doc["software"] = "DeviceHub"
+            doc["CUSTOMER_ID"] = tag
             doc["type"] = "WebSnapshot"
         
 

device/templates/details.html

@@ -40,14 +40,6 @@
   <div class="tab-pane fade show active" id="details">
     <h5 class="card-title">Details</h5>
     <div class="row mb-3">
-      <div class="col-lg-3 col-md-4 label ">
-        (<a href="{% url 'device:edit' object.id %}">Edit Device</a>)
-      </div>
-      <div class="col-lg-9 col-md-8">
-      </div>
-    </div>
-
-    <div class="row">
       <div class="col-lg-3 col-md-4 label ">Phid</div>
       <div class="col-lg-9 col-md-8">{{ object.id }}</div>
     </div>

evidence/forms.py

@@ -1,10 +1,16 @@
 import json
+import uuid
+import hashlib
+import datetime
+import pandas as pd
 
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 from utils.forms import MultipleFileField
+from device.models import Device
 from evidence.parse import Build
+from evidence.xapian import index
 from evidence.models import Annotation
 
 
@@ -73,3 +79,122 @@ class UserTagForm(forms.Form):
             key='CUSTOM_ID',
             value=self.tag
         )
+
+
+class ImportForm(forms.Form):
+    file_import = forms.FileField(label=_("File to import"))
+
+    def __init__(self, *args, **kwargs):
+
+        self.rows = []
+        self.properties = {}
+        self.user = kwargs.pop('user')
+        super().__init__(*args, **kwargs)
+
+    def clean_file_import(self):
+        data = self.cleaned_data["file_import"]
+
+        self.file_name = data.name
+        df = pd.read_excel(data)
+        df.fillna('', inplace=True)
+
+        data_pd = df.to_dict(orient='index')
+
+        if not data_pd or df.last_valid_index() is None:
+            self.exception(_("The file you try to import is empty!"))
+
+        for n in data_pd.keys():
+            # import pdb; pdb.set_trace()
+            if 'type' not in [x.lower() for x in data_pd[n]]:
+                raise ValidationError("You need a column with name 'type'")
+
+        
+            for k, v in data_pd[n].items():
+                if k.lower() == "type":
+                    if v not in Device.Types.values:
+                        raise ValidationError("{} is not a valid device".format(v))
+                
+            self.rows.append(data_pd[n])
+
+        return data
+
+    def save(self, commit=True):
+        table = []
+        for row in self.rows:
+            table.append(self.create_annotation(row))
+
+        if commit:
+            for doc, cred in table:
+              cred.save()
+              self.index(doc)
+            return table
+
+        return
+
+    def create_annotation(self, row):
+        doc = self.create_doc(row)
+        if not doc:
+            return []
+
+        data = {
+            'uuid': doc['uuid'],
+            'owner': self.user,
+            'type': Annotation.Type.SYSTEM,
+            'key': 'CUSTOM_ID',
+            'value': doc['CUSTOMER_ID'],
+        }
+            
+        return [doc, Annotation(**data)]
+
+    def index(self, doc):
+        _uuid = doc['uuid']
+        ev = json.dumps(doc)
+        index(_uuid, ev)
+
+    def create_doc(self, row):
+        doc = {}
+        device = {"manufacturer": "", "model": ""}
+        kv = {}
+        _uuid = str(uuid.uuid4())
+        tag = hashlib.sha3_256(_uuid.encode()).hexdigest()
+        
+
+        for k, v in row.items():
+            if k.upper() == "CUSTOM_ID":
+                tag = v
+
+            if not v:
+                continue
+            
+            if k.lower() == "type":
+                device["type"] = v
+            elif k.lower() == "amount":
+                try:
+                    device["amount"] = int(v)
+                except Exception:
+                    device["amount"] = 1
+                    
+            else:
+                kv[k] = v
+
+        if 'amount' not in row.keys():
+            device["amount"] = 1
+                
+        if not device:
+            return
+        
+        doc["device"] = device
+
+        if kv:
+            doc["kv"] = kv
+
+        date = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        
+        if doc:
+            doc["uuid"] = _uuid
+            doc["endTime"] = date
+            doc["software"] = "DeviceHub"
+            doc["CUSTOMER_ID"] = tag
+            doc["type"] = "WebSnapshot"
+
+        return doc

evidence/urls.py

@@ -16,6 +16,7 @@ app_name = 'evidence'
 urlpatterns = [
     path("", views.ListEvidencesView.as_view(), name="list"),
     path("upload", views.UploadView.as_view(), name="upload"),
+    path("import", views.ImportView.as_view(), name="import"),
     path("<uuid:pk>", views.EvidenceView.as_view(), name="details"),
     path("<uuid:pk>/download", views.DownloadEvidenceView.as_view(), name="download"),
 ]

evidence/views.py

@@ -10,7 +10,7 @@ from django.views.generic.edit import (
 
 from dashboard.mixins import  DashboardView, Http403
 from evidence.models import Evidence
-from evidence.forms import UploadForm, UserTagForm
+from evidence.forms import UploadForm, UserTagForm, ImportForm
 # from django.shortcuts import render
 # from rest_framework import viewsets
 # from snapshot.serializers import SnapshotSerializer
@@ -55,6 +55,29 @@ class UploadView(DashboardView, FormView):
         return response
 
 
+class ImportView(DashboardView, FormView):
+    template_name = "upload.html"
+    section = "evidences"
+    title = _("Import Evidence")
+    breadcrumb = "Evidences / Import"
+    success_url = reverse_lazy('evidence:list')
+    form_class = ImportForm
+
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs['user'] = self.request.user
+        return kwargs
+
+    def form_valid(self, form):
+        form.save()
+        response = super().form_valid(form)
+        return response
+
+    def form_invalid(self, form):
+        response = super().form_invalid(form)
+        return response
+
+
 class EvidenceView(DashboardView, FormView):
     template_name = "ev_details.html"
     section = "evidences"

requirements.txt

@@ -3,5 +3,9 @@ Django==5.0.6
 django-bootstrap5==24.2
 django-extensions==3.2.3
 djangorestframework==3.15.1
-py-dmidecode==0.1.3
 python-decouple==3.3
+py-dmidecode==0.1.3
+pandas==2.2.2
+xlrd==2.0.1
+odfpy==1.4.1
+

reset.sh

@@ -1,4 +1,4 @@
 rm db/*
 ./manage.py migrate
 ./manage.py add_user user@example.org 1234
-./manage.py up_snapshots example/ user@example.org
+./manage.py up_snapshots example/snapshots/ user@example.org