xapian #1

Merged
cayop merged 26 commits from xapian into master 2024-09-17 10:11:28 +00:00
10 changed files with 160 additions and 15 deletions
Showing only changes of commit e2f9855954 - Show all commits

View File

@ -132,7 +132,7 @@
</a> </a>
<ul class="flex-column mb-2 ul_sidebar accordion-collapse {% if section == 'People' %}expanded{% else %}collapse{% endif %}" id="ul_placeholders" data-bs-parent="#sidebarMenu"> <ul class="flex-column mb-2 ul_sidebar accordion-collapse {% if section == 'People' %}expanded{% else %}collapse{% endif %}" id="ul_placeholders" data-bs-parent="#sidebarMenu">
<li class="nav-item"> <li class="nav-item">
<a class="nav-link{% if path == 'admin_people_list' %} active2{% endif %}" href="{# url 'idhub:admin_people_list' #}"> <a class="nav-link{% if path == 'admin_people_list' %} active2{% endif %}" href="{% url 'evidence:import' %}">
{% trans 'Upload Spreadsheet' %} {% trans 'Upload Spreadsheet' %}
</a> </a>
</li> </li>

View File

@ -5,7 +5,7 @@ import datetime
from django import forms from django import forms
from evidence.models import Annotation from evidence.models import Annotation
from evidence.xapian import search, index from evidence.xapian import index
DEVICE_TYPES = [ DEVICE_TYPES = [
("Desktop", "Desktop"), ("Desktop", "Desktop"),
@ -62,8 +62,6 @@ class BaseDeviceFormSet(forms.BaseFormSet):
if not device: if not device:
return return
device["manufacturer"] = ""
device["model"] = tag
doc["device"] = device doc["device"] = device
@ -71,10 +69,12 @@ class BaseDeviceFormSet(forms.BaseFormSet):
doc["kv"] = kv doc["kv"] = kv
date = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") date = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
if doc: if doc:
doc["uuid"] = self.uuid doc["uuid"] = self.uuid
doc["endTime"] = date doc["endTime"] = date
doc["software"] = "DeviceHub" doc["software"] = "DeviceHub"
doc["CUSTOMER_ID"] = tag
doc["type"] = "WebSnapshot" doc["type"] = "WebSnapshot"

View File

@ -40,14 +40,6 @@
<div class="tab-pane fade show active" id="details"> <div class="tab-pane fade show active" id="details">
<h5 class="card-title">Details</h5> <h5 class="card-title">Details</h5>
<div class="row mb-3"> <div class="row mb-3">
<div class="col-lg-3 col-md-4 label ">
(<a href="{% url 'device:edit' object.id %}">Edit Device</a>)
</div>
<div class="col-lg-9 col-md-8">
</div>
</div>
<div class="row">
<div class="col-lg-3 col-md-4 label ">Phid</div> <div class="col-lg-3 col-md-4 label ">Phid</div>
<div class="col-lg-9 col-md-8">{{ object.id }}</div> <div class="col-lg-9 col-md-8">{{ object.id }}</div>
</div> </div>

View File

@ -1,10 +1,16 @@
import json import json
import uuid
import hashlib
import datetime
import pandas as pd
from django import forms from django import forms
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from utils.forms import MultipleFileField from utils.forms import MultipleFileField
from device.models import Device
from evidence.parse import Build from evidence.parse import Build
from evidence.xapian import index
from evidence.models import Annotation from evidence.models import Annotation
@ -73,3 +79,122 @@ class UserTagForm(forms.Form):
key='CUSTOM_ID', key='CUSTOM_ID',
value=self.tag value=self.tag
) )
class ImportForm(forms.Form):
file_import = forms.FileField(label=_("File to import"))
def __init__(self, *args, **kwargs):
self.rows = []
self.properties = {}
self.user = kwargs.pop('user')
super().__init__(*args, **kwargs)
def clean_file_import(self):
data = self.cleaned_data["file_import"]
self.file_name = data.name
df = pd.read_excel(data)
df.fillna('', inplace=True)
data_pd = df.to_dict(orient='index')
if not data_pd or df.last_valid_index() is None:
self.exception(_("The file you try to import is empty!"))
for n in data_pd.keys():
# import pdb; pdb.set_trace()
if 'type' not in [x.lower() for x in data_pd[n]]:
raise ValidationError("You need a column with name 'type'")
for k, v in data_pd[n].items():
if k.lower() == "type":
if v not in Device.Types.values:
raise ValidationError("{} is not a valid device".format(v))
self.rows.append(data_pd[n])
return data
def save(self, commit=True):
table = []
for row in self.rows:
table.append(self.create_annotation(row))
if commit:
for doc, cred in table:
cred.save()
self.index(doc)
return table
return
def create_annotation(self, row):
doc = self.create_doc(row)
if not doc:
return []
data = {
'uuid': doc['uuid'],
'owner': self.user,
'type': Annotation.Type.SYSTEM,
'key': 'CUSTOM_ID',
'value': doc['CUSTOMER_ID'],
}
return [doc, Annotation(**data)]
def index(self, doc):
_uuid = doc['uuid']
ev = json.dumps(doc)
index(_uuid, ev)
def create_doc(self, row):
doc = {}
device = {"manufacturer": "", "model": ""}
kv = {}
_uuid = str(uuid.uuid4())
tag = hashlib.sha3_256(_uuid.encode()).hexdigest()
for k, v in row.items():
if k.upper() == "CUSTOM_ID":
tag = v
if not v:
continue
if k.lower() == "type":
device["type"] = v
elif k.lower() == "amount":
try:
device["amount"] = int(v)
except Exception:
device["amount"] = 1
else:
kv[k] = v
if 'amount' not in row.keys():
device["amount"] = 1
if not device:
return
doc["device"] = device
if kv:
doc["kv"] = kv
date = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
if doc:
doc["uuid"] = _uuid
doc["endTime"] = date
doc["software"] = "DeviceHub"
doc["CUSTOMER_ID"] = tag
doc["type"] = "WebSnapshot"
return doc

View File

@ -16,6 +16,7 @@ app_name = 'evidence'
urlpatterns = [ urlpatterns = [
path("", views.ListEvidencesView.as_view(), name="list"), path("", views.ListEvidencesView.as_view(), name="list"),
path("upload", views.UploadView.as_view(), name="upload"), path("upload", views.UploadView.as_view(), name="upload"),
path("import", views.ImportView.as_view(), name="import"),
path("<uuid:pk>", views.EvidenceView.as_view(), name="details"), path("<uuid:pk>", views.EvidenceView.as_view(), name="details"),
path("<uuid:pk>/download", views.DownloadEvidenceView.as_view(), name="download"), path("<uuid:pk>/download", views.DownloadEvidenceView.as_view(), name="download"),
] ]

View File

@ -10,7 +10,7 @@ from django.views.generic.edit import (
from dashboard.mixins import DashboardView, Http403 from dashboard.mixins import DashboardView, Http403
from evidence.models import Evidence from evidence.models import Evidence
from evidence.forms import UploadForm, UserTagForm from evidence.forms import UploadForm, UserTagForm, ImportForm
# from django.shortcuts import render # from django.shortcuts import render
# from rest_framework import viewsets # from rest_framework import viewsets
# from snapshot.serializers import SnapshotSerializer # from snapshot.serializers import SnapshotSerializer
@ -55,6 +55,29 @@ class UploadView(DashboardView, FormView):
return response return response
class ImportView(DashboardView, FormView):
template_name = "upload.html"
section = "evidences"
title = _("Import Evidence")
breadcrumb = "Evidences / Import"
success_url = reverse_lazy('evidence:list')
form_class = ImportForm
def get_form_kwargs(self):
kwargs = super().get_form_kwargs()
kwargs['user'] = self.request.user
return kwargs
def form_valid(self, form):
form.save()
response = super().form_valid(form)
return response
def form_invalid(self, form):
response = super().form_invalid(form)
return response
class EvidenceView(DashboardView, FormView): class EvidenceView(DashboardView, FormView):
template_name = "ev_details.html" template_name = "ev_details.html"
section = "evidences" section = "evidences"

BIN
example/placeholders1.ods Normal file

Binary file not shown.

View File

@ -3,5 +3,9 @@ Django==5.0.6
django-bootstrap5==24.2 django-bootstrap5==24.2
django-extensions==3.2.3 django-extensions==3.2.3
djangorestframework==3.15.1 djangorestframework==3.15.1
py-dmidecode==0.1.3
python-decouple==3.3 python-decouple==3.3
py-dmidecode==0.1.3
pandas==2.2.2
xlrd==2.0.1
odfpy==1.4.1

View File

@ -1,4 +1,4 @@
rm db/* rm db/*
./manage.py migrate ./manage.py migrate
./manage.py add_user user@example.org 1234 ./manage.py add_user user@example.org 1234
./manage.py up_snapshots example/ user@example.org ./manage.py up_snapshots example/snapshots/ user@example.org