ereuse
/
devicehub-django
8
0
Fork 0
Code Issues 5 Pull Requests 6 Packages Projects Releases Wiki Activity

WIP: rework/properties #31

Draft
rskthomas wants to merge 20 commits from rework/properties into main
pull from: rework/properties
merge into: ereuse:main
Conversation 0 Commits 20 Files Changed 25 +386 -241
25 changed files with 386 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/urls.py
View File

@ -7,7 +7,7 @@ app_name = 'api'
urlpatterns = [
path('v1/snapshot/', views.NewSnapshotView.as_view(), name='new_snapshot'),
path('v1/annotation/<str:pk>/', views.AddAnnotationView.as_view(), name='new_annotation'),
path('v1/property/<str:pk>/', views.AddPropertyView.as_view(), name='new_property'),
path('v1/device/<str:pk>/', views.DetailsDeviceView.as_view(), name='device'),
path('v1/tokens/', views.TokenView.as_view(), name='tokens'),
path('v1/tokens/new', views.TokenNewView.as_view(), name='new_token'),

38
api/views.py
View File

@ -21,7 +21,7 @@ from django.views.generic.edit import (
from utils.save_snapshots import move_json, save_in_disk
from django.views.generic.edit import View
from dashboard.mixins import DashboardView
from evidence.models import Annotation
from evidence.models import SystemProperty, UserProperty, Property
from evidence.parse_details import ParseSnapshot
from evidence.parse import Build
from device.models import Device
@ -90,11 +90,11 @@ class NewSnapshotView(ApiMixing):
logger.error("%s", txt)
return JsonResponse({'status': txt}, status=500)
exist_annotation = Annotation.objects.filter(
exist_property = SystemProperty.objects.filter(
uuid=data['uuid']
).first()
if exist_annotation:
if exist_property:
txt = "error: the snapshot {} exist".format(data['uuid'])
logger.warning("%s", txt)
return JsonResponse({'status': txt}, status=500)
@ -111,25 +111,25 @@ class NewSnapshotView(ApiMixing):
text = "fail: It is not possible to parse snapshot"
return JsonResponse({'status': text}, status=500)
annotation = Annotation.objects.filter(
property = SystemProperty.objects.filter(
uuid=data['uuid'],
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
# TODO this is hardcoded, it should select the user preferred algorithm
key="hidalgo1",
owner=self.tk.owner.institution
).first()
if not annotation:
logger.error("Error: No annotation for uuid: %s", data["uuid"])
if not property:
logger.error("Error: No property for uuid: %s", data["uuid"])
return JsonResponse({'status': 'fail'}, status=500)
url_args = reverse_lazy("device:details", args=(annotation.value,))
url_args = reverse_lazy("device:details", args=(property.value,))
url = request.build_absolute_uri(url_args)
response = {
"status": "success",
"dhid": annotation.value[:6].upper(),
"dhid": property.value[:6].upper(),
"url": url,
# TODO replace with public_url when available
"public_url": url
@ -255,22 +255,21 @@ class DetailsDeviceView(ApiMixing):
"components": snapshot.get("components"),
})
uuids = Annotation.objects.filter(
uuids = SystemProperty.objects.filter(
owner=self.tk.owner.institution,
value=self.pk
).values("uuid")
annotations = Annotation.objects.filter(
properties = UserProperty.objects.filter(
uuid__in=uuids,
owner=self.tk.owner.institution,
type = Annotation.Type.USER
).values_list("key", "value")
data.update({"annotations": list(annotations)})
data.update({"properties": list(properties)})
return data
class AddAnnotationView(ApiMixing):
class AddPropertyView(ApiMixing):
def post(self, request, *args, **kwargs):
response = self.auth()
@ -279,13 +278,13 @@ class AddAnnotationView(ApiMixing):
self.pk = kwargs['pk']
institution = self.tk.owner.institution
self.annotation = Annotation.objects.filter(
self.property = SystemProperty.objects.filter(
owner=institution,
value=self.pk,
type=Annotation.Type.SYSTEM
type=Property.Type.SYSTEM
).first()
if not self.annotation:
if not self.property:
return JsonResponse({}, status=404)
try:
@ -296,10 +295,9 @@ class AddAnnotationView(ApiMixing):
logger.error("Invalid Snapshot of user %s", self.tk.owner)
return JsonResponse({'error': 'Invalid JSON'}, status=500)
Annotation.objects.create(
uuid=self.annotation.uuid,
UserProperty.objects.create(
uuid=self.property.uuid,
owner=self.tk.owner.institution,
type = Annotation.Type.USER,
key = key,
value = value
)

4
dashboard/mixins.py
View File

@ -6,7 +6,7 @@ from django.core.exceptions import PermissionDenied
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic.base import TemplateView
from device.models import Device
from evidence.models import Annotation
from evidence.models import SystemProperty
from lot.models import LotTag
@ -49,7 +49,7 @@ class DashboardView(LoginRequiredMixin):
dev_ids = self.request.session.pop("devices", [])
self._devices = []
for x in Annotation.objects.filter(value__in=dev_ids).filter(
for x in SystemProperty.objects.filter(value__in=dev_ids).filter(
owner=self.request.user.institution
).distinct():
self._devices.append(Device(id=x.value))

4
dashboard/templates/unassigned_devices.html
View File

@ -20,9 +20,9 @@
{% trans 'Exports' %}
</a>
{% if lot %}
<a href="{% url 'lot:annotations' object.id %}" type="button" class="btn btn-green-admin">
<a href="{% url 'lot:properties' object.id %}" type="button" class="btn btn-green-admin">
<i class="bi bi-tag"></i>
{% trans 'Annotations' %}
{% trans 'properties' %}
</a>
{% endif %}
</div>

12
dashboard/views.py
View File

@ -6,7 +6,7 @@ from django.shortcuts import Http404
from django.db.models import Q
from dashboard.mixins import InventaryMixin, DetailsMixin
from evidence.models import Annotation
from evidence.models import Property, SystemProperty
from evidence.xapian import search
from device.models import Device
from lot.models import Lot
@ -74,7 +74,7 @@ class SearchView(InventaryMixin):
for x in matches:
# devices.append(self.get_annotations(x))
dev = self.get_annotations(x)
dev = self.get_properties(x)
if dev.id not in dev_id:
devices.append(dev)
dev_id.append(dev.id)
@ -83,10 +83,10 @@ class SearchView(InventaryMixin):
# TODO fix of pagination, the count is not correct
return devices, count
def get_annotations(self, xp):
def get_properties(self, xp):
snap = xp.document.get_data()
uuid = json.loads(snap).get('uuid')
return Device.get_annotation_from_uuid(uuid, self.request.user.institution)
return Device.get_properties_from_uuid(uuid, self.request.user.institution)
def search_hids(self, query, offset, limit):
qry = Q()
@ -95,8 +95,8 @@ class SearchView(InventaryMixin):
if i:
qry |= Q(value__startswith=i)
chids = Annotation.objects.filter(
type=Annotation.Type.SYSTEM,
chids = SystemProperty.objects.filter(
type=Property.Type.SYSTEM,
owner=self.request.user.institution
).filter(
qry

4
device/forms.py
View File

@ -1,5 +1,5 @@
from django import forms
from utils.device import create_annotation, create_doc, create_index
from utils.device import create_property, create_doc, create_index
from utils.save_snapshots import move_json, save_in_disk
@ -59,7 +59,7 @@ class BaseDeviceFormSet(forms.BaseFormSet):
path_name = save_in_disk(doc, self.user.institution.name, place="placeholder")
create_index(doc, self.user)
create_annotation(doc, user, commit=commit)
create_property(doc, user, commit=commit)
move_json(path_name, self.user.institution.name, place="placeholder")
return doc

104
device/models.py
View File

@ -1,7 +1,7 @@
from django.db import models, connection
from utils.constants import ALGOS
from evidence.models import Annotation, Evidence
from evidence.models import SystemProperty, UserProperty, Property, Evidence
from lot.models import DeviceLot
@ -29,7 +29,7 @@ class Device:
self.shortid = self.pk[:6].upper()
self.algorithm = None
self.owner = None
self.annotations = []
self.properties = []
self.hids = []
self.uuids = []
self.evidences = []
@ -38,61 +38,60 @@ class Device:
self.get_last_evidence()
def initial(self):
self.get_annotations()
self.get_properties()
self.get_uuids()
self.get_hids()
self.get_evidences()
self.get_lots()
def get_annotations(self):
if self.annotations:
return self.annotations
def get_properties(self):
if self.properties:
return self.properties
self.annotations = Annotation.objects.filter(
type=Annotation.Type.SYSTEM,
self.properties = SystemProperty.objects.filter(
type=Property.Type.SYSTEM,
value=self.id
).order_by("-created")
if self.annotations.count():
self.algorithm = self.annotations[0].key
self.owner = self.annotations[0].owner
if self.properties.count():
self.algorithm = self.properties[0].key
self.owner = self.properties[0].owner
return self.annotations
return self.properties
def get_user_annotations(self):
def get_user_properties(self):
if not self.uuids:
self.get_uuids()
annotations = Annotation.objects.filter(
user_properties = UserProperty.objects.filter(
uuid__in=self.uuids,
owner=self.owner,
type=Annotation.Type.USER
)
return annotations
return user_properties
def get_user_documents(self):
if not self.uuids:
self.get_uuids()
annotations = Annotation.objects.filter(
properties = SystemProperty.objects.filter(
uuid__in=self.uuids,
owner=self.owner,
type=Annotation.Type.DOCUMENT
type=Property.Type.DOCUMENT
)
return annotations
return properties
def get_uuids(self):
for a in self.get_annotations():
for a in self.get_properties():
if a.uuid not in self.uuids:
self.uuids.append(a.uuid)
def get_hids(self):
annotations = self.get_annotations()
properties = self.get_properties()
algos = list(ALGOS.keys())
algos.append('CUSTOM_ID')
self.hids = list(set(annotations.filter(
type=Annotation.Type.SYSTEM,
self.hids = list(set(properties.filter(
type=Property.Type.SYSTEM,
key__in=algos,
).values_list("value", flat=True)))
@ -103,11 +102,12 @@ class Device:
self.evidences = [Evidence(u) for u in self.uuids]
def get_last_evidence(self):
annotations = self.get_annotations()
if not annotations.count():
properties = self.get_properties()
if not properties.count():
return
annotation = annotations.first()
self.last_evidence = Evidence(annotation.uuid)
property = properties.first()
self.last_evidence = Evidence(property.uuid)
def is_eraseserver(self):
if not self.uuids:
@ -115,13 +115,13 @@ class Device:
if not self.uuids:
return False
annotation = Annotation.objects.filter(
property = SystemProperty.objects.filter(
uuid__in=self.uuids,
owner=self.owner,
type=Annotation.Type.ERASE_SERVER
type=Property.Type.ERASE_SERVER
).first()
if annotation:
if property:
return True
return False
@ -136,7 +136,7 @@ class Device:
def get_unassigned(cls, institution, offset=0, limit=None):
sql = """
WITH RankedAnnotations AS (
WITH RankedProperties AS (
SELECT
t1.value,
t1.key,
@ -149,8 +149,8 @@ class Device:
ELSE 3
END,
t1.created DESC
) AS row_num
FROM evidence_annotation AS t1
) AS row_num
FROM evidence_systemproperty AS t1
LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
WHERE t2.device_id IS NULL
AND t1.owner_id = {institution}
@ -159,24 +159,24 @@ class Device:
SELECT DISTINCT
value
FROM
RankedAnnotations
RankedProperties
WHERE
row_num = 1
""".format(
institution=institution.id,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
)
if limit:
sql += " limit {} offset {}".format(int(limit), int(offset))
sql += ";"
annotations = []
properties = []
with connection.cursor() as cursor:
cursor.execute(sql)
annotations = cursor.fetchall()
properties = cursor.fetchall()
devices = [cls(id=x[0]) for x in annotations]
devices = [cls(id=x[0]) for x in properties]
count = cls.get_unassigned_count(institution)
return devices, count
@ -184,7 +184,7 @@ class Device:
def get_unassigned_count(cls, institution):
sql = """
WITH RankedAnnotations AS (
WITH RankedProperties AS (
SELECT
t1.value,
t1.key,
@ -198,30 +198,30 @@ class Device:
END,
t1.created DESC
) AS row_num
FROM evidence_annotation AS t1
FROM evidence_systemproperty AS t1
LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
WHERE t2.device_id IS NULL
AND t1.owner_id = {institution}
AND t1.type = {type}
And t1.type = '{type}'
)
SELECT
COUNT(DISTINCT value)
FROM
RankedAnnotations
RankedProperties
WHERE
row_num = 1
""".format(
institution=institution.id,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
)
with connection.cursor() as cursor:
cursor.execute(sql)
return cursor.fetchall()[0][0]
@classmethod
def get_annotation_from_uuid(cls, uuid, institution):
def get_properties_from_uuid(cls, uuid, institution):
sql = """
WITH RankedAnnotations AS (
WITH RankedProperties AS (
SELECT
t1.value,
t1.key,
@ -235,31 +235,31 @@ class Device:
END,
t1.created DESC
) AS row_num
FROM evidence_annotation AS t1
FROM evidence_systemproperty AS t1
LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id
WHERE t2.device_id IS NULL
AND t1.owner_id = {institution}
AND t1.type = {type}
AND t1.type = '{type}'
AND t1.uuid = '{uuid}'
)
SELECT DISTINCT
value
FROM
RankedAnnotations
RankedProperties
WHERE
row_num = 1;
""".format(
uuid=uuid.replace("-", ""),
institution=institution.id,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
)
annotations = []
properties = []
with connection.cursor() as cursor:
cursor.execute(sql)
annotations = cursor.fetchall()
properties = cursor.fetchall()
return cls(id=annotations[0][0])
return cls(id=properties[0][0])
@property
def is_websnapshot(self):

82
device/templates/details.html
View File

@ -15,7 +15,7 @@
<a href="#details" class="nav-link active" data-bs-toggle="tab" data-bs-target="#details">{% trans 'General details' %}</a>
</li>
<li class="nav-item">
<a href="#annotations" class="nav-link" data-bs-toggle="tab" data-bs-target="#annotations">{% trans 'User annotations' %}</a>
<a href="#user_properties" class="nav-link" data-bs-toggle="tab" data-bs-target="#user_properties">{% trans 'User properties' %}</a>
</li>
<li class="nav-item">
<a href="#documents" class="nav-link" data-bs-toggle="tab" data-bs-target="#documents">{% trans 'Documents' %}</a>
@ -100,15 +100,15 @@
{% endfor %}
</div>
<div class="tab-pane fade" id="annotations">
<div class="tab-pane fade" id="user_properties">
<div class="btn-group mt-1 mb-3">
<a href="{% url 'device:add_annotation' object.pk %}" class="btn btn-primary">
<a href="{% url 'device:add_user_property' object.pk %}" class="btn btn-primary">
<i class="bi bi-plus"></i>
{% trans 'Add new annotation' %}
{% trans 'New user property' %}
</a>
</div>
<h5 class="card-title">{% trans 'Annotations' %}</h5>
<h5 class="card-title">{% trans 'User properties' %}</h5>
<table class="table table-striped">
<thead>
<tr>
@ -122,23 +122,87 @@
{% trans 'Created on' %}
</th>
<th></th>
<th></th>
</tr>
</thead>
<tbody>
{% for a in object.get_user_annotations %}
{% for a in object.get_user_properties %}
<tr>
<td>{{ a.key }}</td>
<td>{{ a.value }}</td>
<td>{{ a.created }}</td>
<td></td>
<td></td>
<td>
<div class="btn-group float-end">
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#editModal{{ a.id }}">
<i class="bi bi-pencil"></i> {% trans 'Edit' %}
</button>
<button type="button" class="btn btn-sm btn-danger" data-bs-toggle="modal" data-bs-target="#deleteModal{{ a.id }}">
<i class="bi bi-trash"></i>
</button>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<!-- pop up modal for delete confirmation -->
{% for a in object.get_user_properties %}
<div class="modal fade" id="deleteModal{{ a.id }}" tabindex="-1" aria-labelledby="deleteModalLabel{{ a.id }}" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="deleteModalLabel{{ a.id }}">{% trans "Confirm Deletion" %}</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p><strong>{% trans "Key:" %}</strong> {{ a.key }}</p>
<p><strong>{% trans "Value:" %}</strong> {{ a.value }}</p>
<p><strong>{% trans "Created on:" %}</strong> {{ a.created }}</p>
</div>
<div class="modal-footer justify-content-center">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{% trans "Cancel" %}</button>
<form method="post" action="{% url 'device:delete_user_property' a.id %}">
{% csrf_token %}
<button type="submit" class="btn btn-danger">{% trans "Delete" %}</button>
</form>
</div>
</div>
</div>
</div>
{% endfor %}
<!-- popup modals for edit button -->
{% for a in object.get_user_properties %}
<div class="modal fade" id="editModal{{ a.id }}" tabindex="-1" aria-labelledby="editModalLabel{{ a.id }}" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="editModalLabel{{ a.id }}">{% trans "Edit User Property" %}</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<form id="editForm{{ a.id }}" method="post" action="{% url 'device:update_user_property' a.id %}">
{% csrf_token %}
<div class="mb-3">
<label for="key" class="form-label">{% trans "Key" %}</label>
<input type="text" class="form-control" id="key" name="key" value="{{ a.key }}">
</div>
<div class="mb-3">
<label for="value" class="form-label">{% trans "Value" %}</label>
<input type="text" class="form-control" id="value" name="value" value="{{ a.value }}">
</div>
<div class="modal-footer justify-content-center">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{% trans "Cancel" %}</button>
<button type="submit" class="btn btn-primary">{% trans "Save changes" %}</button>
</div>
</form>
</div>
</div>
</div>
</div>
{% endfor %}
<div class="tab-pane fade" id="documents">
<div class="btn-group mt-1 mb-3">
<a href="{% url 'device:add_document' object.pk %}" class="btn btn-primary">

0
device/templates/new_annotation.html → device/templates/new_user_property.html
View File

4
device/urls.py
View File

@ -7,7 +7,9 @@ urlpatterns = [
path("add/", views.NewDeviceView.as_view(), name="add"),
path("edit/<str:pk>/", views.EditDeviceView.as_view(), name="edit"),
path("<str:pk>/", views.DetailsView.as_view(), name="details"),
path("<str:pk>/annotation/add", views.AddAnnotationView.as_view(), name="add_annotation"),
path("<str:pk>/user_property/add", views.AddUserPropertyView.as_view(), name="add_user_property"),
path("user_property/<int:pk>/delete", views.DeleteUserPropertyView.as_view(), name="delete_user_property"),
path("user_property/<int:pk>/update", views.UpdateUserPropertyView.as_view(), name="update_user_property"),
path("<str:pk>/document/add", views.AddDocumentView.as_view(), name="add_document"),
path("<str:pk>/public/", views.PublicDeviceWebView.as_view(), name="device_web"),

110
device/views.py
View File

@ -1,23 +1,28 @@
import json
import logging
from django.http import JsonResponse
from django.http import Http404
from django.urls import reverse_lazy
from django.shortcuts import get_object_or_404, Http404
from django.contrib import messages
from django.shortcuts import get_object_or_404, redirect, Http404
from django.utils.translation import gettext_lazy as _
from django.views.generic.edit import (
CreateView,
UpdateView,
FormView,
DeleteView,
)
from django.views.generic.base import TemplateView
from dashboard.mixins import DashboardView, Http403
from evidence.models import Annotation
from evidence.models import UserProperty, SystemProperty, Property
from lot.models import LotTag
from device.models import Device
from device.forms import DeviceFormSet
device_logger = logging.getLogger('device_log')
class NewDeviceView(DashboardView, FormView):
template_name = "new_device.html"
title = _("New Device")
@ -69,7 +74,7 @@ class EditDeviceView(DashboardView, UpdateView):
title = _("Update Device")
breadcrumb = "Device / Update Device"
success_url = reverse_lazy('dashboard:unassigned_devices')
model = Annotation
model = SystemProperty
def get_form_kwargs(self):
pk = self.kwargs.get('pk')
@ -87,7 +92,7 @@ class DetailsView(DashboardView, TemplateView):
template_name = "details.html"
title = _("Device")
breadcrumb = "Device / Details"
model = Annotation
model = SystemProperty
def get(self, request, *args, **kwargs):
self.pk = kwargs['pk']
@ -167,65 +172,128 @@ class PublicDeviceWebView(TemplateView):
return JsonResponse(device_data)
class AddAnnotationView(DashboardView, CreateView):
template_name = "new_annotation.html"
title = _("New annotation")
breadcrumb = "Device / New annotation"
class AddUserPropertyView(DashboardView, CreateView):
template_name = "new_user_property.html"
title = _("New User Property")
breadcrumb = "Device / New Property"
success_url = reverse_lazy('dashboard:unassigned_devices')
model = Annotation
model = UserProperty
fields = ("key", "value")
def form_valid(self, form):
form.instance.owner = self.request.user.institution
form.instance.user = self.request.user
form.instance.uuid = self.annotation.uuid
form.instance.type = Annotation.Type.USER
form.instance.uuid = self.property.uuid
form.instance.type = Property.Type.USER
messages.success(self.request, _("User property successfully added."))
response = super().form_valid(form)
return response
def get_form_kwargs(self):
pk = self.kwargs.get('pk')
institution = self.request.user.institution
self.annotation = Annotation.objects.filter(
self.property = SystemProperty.objects.filter(
owner=institution,
value=pk,
type=Annotation.Type.SYSTEM
type=Property.Type.SYSTEM
).first()
if not self.annotation:
if not self.property:
raise Http404
self.success_url = reverse_lazy('device:details', args=[pk])
kwargs = super().get_form_kwargs()
return kwargs
class UpdateUserPropertyView(DashboardView, UpdateView):
template_name = "new_user_property.html"
title = _("Update User Property")
breadcrumb = "Device / Update Property"
model = UserProperty
fields = ("key", "value")
def get_form_kwargs(self):
pk = self.kwargs.get('pk')
user_property = get_object_or_404(UserProperty, pk=pk, owner=self.request.user.institution)
if not user_property:
raise Http404
kwargs = super().get_form_kwargs()
kwargs['instance'] = user_property
return kwargs
def form_valid(self, form):
old_key= self.object.key
old_value = self.object.value
new_key = form.cleaned_data['key']
new_value = form.cleaned_data['value']
form.instance.owner = self.request.user.institution
form.instance.user = self.request.user
form.instance.type = Property.Type.USER
response = super().form_valid(form)
messages.success(self.request, _("User property updated successfully."))
device_logger.info(
f"Updated property from (key='{old_key}', value='{old_value}') to (key='{new_key}', value='{new_value}') by user {self.request.user}."
)
return response
def get_success_url(self):
return self.request.META.get('HTTP_REFERER', reverse_lazy('device:details', args=[self.object.pk]))
class DeleteUserPropertyView(DashboardView, DeleteView):
model = UserProperty
def post(self, request, *args, **kwargs):
self.pk = kwargs['pk']
referer = request.META.get('HTTP_REFERER')
if not referer:
raise Http404("No referer header found")
self.object = get_object_or_404(
self.model,
pk=self.pk,
owner=self.request.user.institution
)
old_value = self.object.key
self.object.delete()
device_logger.info(f"Deleted property with key '{old_value}' by user {self.request.user}.")
messages.success(self.request, _("User property deleted successfully."))
# Redirect back to the original URL
return redirect(referer)
class AddDocumentView(DashboardView, CreateView):
template_name = "new_annotation.html"
template_name = "new_user_property.html"
title = _("New Document")
breadcrumb = "Device / New document"
success_url = reverse_lazy('dashboard:unassigned_devices')
model = Annotation
model = SystemProperty
fields = ("key", "value")
def form_valid(self, form):
form.instance.owner = self.request.user.institution
form.instance.user = self.request.user
form.instance.uuid = self.annotation.uuid
form.instance.type = Annotation.Type.DOCUMENT
form.instance.uuid = self.property.uuid
form.instance.type = Property.Type.DOCUMENT
response = super().form_valid(form)
return response
def get_form_kwargs(self):
pk = self.kwargs.get('pk')
institution = self.request.user.institution
self.annotation = Annotation.objects.filter(
self.property = SystemProperty.objects.filter(
owner=institution,
value=pk,
type=Annotation.Type.SYSTEM
type=Property.Type.SYSTEM
).first()
if not self.annotation:
if not self.property:
raise Http404
self.success_url = reverse_lazy('device:details', args=[pk])

17
dhub/settings.py
View File

@ -210,6 +210,10 @@ LOGGING = {
'()': CustomFormatter,
'format': '%(levelname)s %(asctime)s %(message)s'
},
'verbose': {
'format': '{levelname} {asctime} {module} {message}',
'style': '{',
},
},
"handlers": {
"console": {
@ -217,6 +221,12 @@ LOGGING = {
"class": "logging.StreamHandler",
"formatter": "colored"
},
'device_log_file': {
'level': 'INFO',
'class': 'logging.FileHandler',
'filename': '/var/log/device_changes.log',
'formatter': 'verbose',
},
},
"root": {
"handlers": ["console"],
@ -232,7 +242,12 @@ LOGGING = {
"handlers": ["console"],
"level": "ERROR",
"propagate": False,
}
},
'device_log': {
'handlers': ['device_log_file'],
'level': 'INFO',
'propagate': False,
},
}
}

36
evidence/forms.py
View File

@ -4,11 +4,11 @@ import pandas as pd
from django import forms
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from utils.device import create_annotation, create_doc, create_index
from utils.device import create_property, create_doc, create_index
from utils.forms import MultipleFileField
from device.models import Device
from evidence.parse import Build
from evidence.models import Annotation
from evidence.models import SystemProperty, Property
from utils.save_snapshots import move_json, save_in_disk
@ -30,11 +30,11 @@ class UploadForm(forms.Form):
try:
file_json = json.loads(file_data)
Build(file_json, None, check=True)
exist_annotation = Annotation.objects.filter(
exists_property = SystemProperty.objects.filter(
uuid=file_json['uuid']
).first()
if exist_annotation:
if exists_property:
raise ValidationError(
_("The snapshot already exists"),
code="duplicate_snapshot",
@ -68,9 +68,9 @@ class UserTagForm(forms.Form):
self.pk = None
self.uuid = kwargs.pop('uuid', None)
self.user = kwargs.pop('user')
instance = Annotation.objects.filter(
instance = SystemProperty.objects.filter(
uuid=self.uuid,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
key='CUSTOM_ID',
owner=self.user.institution
).first()
@ -86,9 +86,9 @@ class UserTagForm(forms.Form):
if not data:
return False
self.tag = data
self.instance = Annotation.objects.filter(
self.instance = SystemProperty.objects.filter(
uuid=self.uuid,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
key='CUSTOM_ID',
owner=self.user.institution
).first()
@ -106,9 +106,9 @@ class UserTagForm(forms.Form):
self.instance.save()
return
Annotation.objects.create(
SystemProperty.objects.create(
uuid=self.uuid,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
key='CUSTOM_ID',
value=self.tag,
owner=self.user.institution,
@ -164,8 +164,8 @@ class ImportForm(forms.Form):
table = []
for row in self.rows:
doc = create_doc(row)
annotation = create_annotation(doc, self.user)
table.append((doc, annotation))
property = create_property(doc, self.user)
table.append((doc, property))
if commit:
for doc, cred in table:
@ -186,9 +186,9 @@ class EraseServerForm(forms.Form):
self.pk = None
self.uuid = kwargs.pop('uuid', None)
self.user = kwargs.pop('user')
instance = Annotation.objects.filter(
instance = SystemProperty.objects.filter(
uuid=self.uuid,
type=Annotation.Type.ERASE_SERVER,
type=SystemProperty.Type.ERASE_SERVER,
key='ERASE_SERVER',
owner=self.user.institution
).first()
@ -201,9 +201,9 @@ class EraseServerForm(forms.Form):
def clean(self):
self.erase_server = self.cleaned_data.get('erase_server', False)
self.instance = Annotation.objects.filter(
self.instance = SystemProperty.objects.filter(
uuid=self.uuid,
type=Annotation.Type.ERASE_SERVER,
type=Property.Type.ERASE_SERVER,
key='ERASE_SERVER',
owner=self.user.institution
).first()
@ -222,9 +222,9 @@ class EraseServerForm(forms.Form):
if self.instance:
return
Annotation.objects.create(
SystemProperty.objects.create(
uuid=self.uuid,
type=Annotation.Type.ERASE_SERVER,
type=Property.Type.ERASE_SERVER,
key='ERASE_SERVER',
value=self.erase_server,
owner=self.user.institution,

4
evidence/management/commands/reindex.py
View File

@ -5,7 +5,7 @@ import logging
from django.core.management.base import BaseCommand
from django.conf import settings
from utils.device import create_annotation, create_doc, create_index
from utils.device import create_property, create_doc, create_index
from user.models import Institution
from evidence.parse import Build
@ -70,7 +70,7 @@ class Command(BaseCommand):
def build_placeholder(self, s, user, f_path):
try:
create_index(s, user)
create_annotation(s, user, commit=True)
create_property(s, user, commit=True)
except Exception as err:
txt = "In placeholder %s \n%s"
logger.warning(txt, f_path, err)

57
evidence/models.py
View File

@ -3,13 +3,15 @@ import json
from dmidecode import DMIParse
from django.db import models
from django.db.models import Q
from utils.constants import STR_EXTEND_SIZE, CHASSIS_DH
from evidence.xapian import search
from evidence.parse_details import ParseSnapshot
from user.models import User, Institution
class Annotation(models.Model):
#TODO: base class is abstract; revise if should be for query efficiency
class Property(models.Model):
class Type(models.IntegerChoices):
SYSTEM = 0, "System"
USER = 1, "User"
@ -26,11 +28,42 @@ class Annotation(models.Model):
value = models.CharField(max_length=STR_EXTEND_SIZE)
class Meta:
#Only for shared behaviour, it is not a table
abstract = True
class SystemProperty(Property):
class Meta:
constraints = [
models.CheckConstraint(
check=~Q(type=1), #Enforce that type is not User
name='property_cannot_be_user'
),
]
#Django orm wont inherit constraints to child
#TODO: check if this is needed
constraints = [
models.UniqueConstraint(
fields=["type", "key", "uuid"], name="unique_type_key_uuid")
fields=["type", "key", "uuid"], name="system_unique_type_key_uuid")
]
class UserProperty(Property):
type = models.SmallIntegerField(default=Property.Type.USER)
class Meta:
constraints = [
models.CheckConstraint(
check=Q(type=1), #Enforce that type is User
name='property_needs_to_be_user'
),
]
constraints = [
models.UniqueConstraint(
fields=["type", "key", "uuid"], name="user_unique_type_key_uuid")
]
class Evidence:
def __init__(self, uuid):
@ -39,22 +72,22 @@ class Evidence:
self.doc = None
self.created = None
self.dmi = None
self.annotations = []
self.properties = []
self.components = []
self.default = "n/a"
self.get_owner()
self.get_time()
def get_annotations(self):
self.annotations = Annotation.objects.filter(
def get_properties(self):
self.properties = SystemProperty.objects.filter(
uuid=self.uuid
).order_by("created")
def get_owner(self):
if not self.annotations:
self.get_annotations()
a = self.annotations.first()
if not self.properties:
self.get_properties()
a = self.properties.first()
if a:
self.owner = a.owner
@ -80,7 +113,7 @@ class Evidence:
self.created = self.doc.get("endTime")
if not self.created:
self.created = self.annotations.last().created
self.created = self.properties.last().created
def get_components(self):
if self.is_legacy():
@ -131,9 +164,9 @@ class Evidence:
@classmethod
def get_all(cls, user):
return Annotation.objects.filter(
return SystemProperty.objects.filter(
owner=user.institution,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
key="hidalgo1",
).order_by("-created").values_list("uuid", "created").distinct()

18
evidence/parse.py
View File

@ -6,7 +6,7 @@ from dmidecode import DMIParse
from json_repair import repair_json
from evidence.parse_details import get_lshw_child
from evidence.models import Annotation
from evidence.models import SystemProperty, Property
from evidence.xapian import index
from utils.constants import CHASSIS_DH
@ -46,7 +46,7 @@ class Build:
return
self.index()
self.create_annotations()
self.create_properties()
def index(self):
snap = json.dumps(self.json)
@ -72,24 +72,24 @@ class Build:
return hashlib.sha3_256(hid.encode()).hexdigest()
def create_annotations(self):
annotation = Annotation.objects.filter(
def create_properties(self):
property = SystemProperty.objects.filter(
uuid=self.uuid,
owner=self.user.institution,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot %s already registered (annotation exists)"
if property:
txt = "Warning: Snapshot %s already registered (property exists)"
logger.warning(txt, self.uuid)
return
for k, v in self.algorithms.items():
Annotation.objects.create(
SystemProperty.objects.create(
uuid=self.uuid,
owner=self.user.institution,
user=self.user,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
key=k,
value=v
)

4
evidence/templates/ev_details.html
View File

@ -45,7 +45,7 @@
</th>
</tr>
</thead>
{% for snap in object.annotations %}
{% for snap in object.properties %}
<tbody>
{% if snap.type == 0 %}
<tr>
@ -94,7 +94,7 @@
</div>
{% if form.tag.value %}
<div class="col-1">
<a class="btn btn-yellow" href="{% url 'evidence:delete_annotation' form.pk %}">{% translate "Delete" %}</a>
<a class="btn btn-yellow" href="{% url 'evidence:delete_user_property' form.pk %}">{% translate "Delete" %}</a>
</div>
{% endif %}
</div>

1
evidence/urls.py
View File

@ -20,5 +20,4 @@ urlpatterns = [
path("<uuid:pk>", views.EvidenceView.as_view(), name="details"),
path("<uuid:pk>/eraseserver", views.EraseServerView.as_view(), name="erase_server"),
path("<uuid:pk>/download", views.DownloadEvidenceView.as_view(), name="download"),
path('annotation/<int:pk>/del', views.AnnotationDeleteView.as_view(), name='delete_annotation'),
]

31
evidence/views.py
View File

@ -13,7 +13,7 @@ from django.views.generic.edit import (
)
from dashboard.mixins import DashboardView, Http403
from evidence.models import Evidence, Annotation
from evidence.models import Property, SystemProperty, UserProperty, Evidence
from evidence.forms import (
UploadForm,
UserTagForm,
@ -95,7 +95,7 @@ class EvidenceView(DashboardView, FormView):
if self.object.owner != self.request.user.institution:
raise Http403
self.object.get_annotations()
self.object.get_properties()
return super().get(request, *args, **kwargs)
def get_context_data(self, **kwargs):
@ -141,31 +141,6 @@ class DownloadEvidenceView(DashboardView, TemplateView):
return response
class AnnotationDeleteView(DashboardView, DeleteView):
model = Annotation
def get(self, request, *args, **kwargs):
self.pk = kwargs['pk']
try:
referer = self.request.META["HTTP_REFERER"]
path_referer = urlparse(referer).path
resolver_match = resolve(path_referer)
url_name = resolver_match.view_name
kwargs_view = resolver_match.kwargs
except:
# if is not possible resolve the reference path return 404
raise Http404
self.object = get_object_or_404(
self.model,
pk=self.pk,
owner=self.request.user.institution
)
self.object.delete()
return redirect(url_name, **kwargs_view)
class EraseServerView(DashboardView, FormView):
@ -182,7 +157,7 @@ class EraseServerView(DashboardView, FormView):
if self.object.owner != self.request.user.institution:
raise Http403
self.object.get_annotations()
self.object.get_properties()
return super().get(request, *args, **kwargs)
def get_context_data(self, **kwargs):

19
lot/models.py
View File

@ -7,8 +7,8 @@ from utils.constants import (
)
from user.models import User, Institution
from device.models import Property
# from device.models import Device
# from evidence.models import Annotation
class LotTag(models.Model):
@ -45,17 +45,8 @@ class Lot(models.Model):
for d in DeviceLot.objects.filter(lot=self, device_id=v):
d.delete()
class LotAnnotation(models.Model):
class Type(models.IntegerChoices):
SYSTEM= 0, "System"
USER = 1, "User"
DOCUMENT = 2, "Document"
created = models.DateTimeField(auto_now_add=True)
class LotProperty (Property):
#uuid is not needed for id
uuid = None
#lot foreign key is
lot = models.ForeignKey(Lot, on_delete=models.CASCADE)
owner = models.ForeignKey(Institution, on_delete=models.CASCADE)
user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True)
type = models.SmallIntegerField(choices=Type)
key = models.CharField(max_length=STR_EXTEND_SIZE)
value = models.CharField(max_length=STR_EXTEND_SIZE)

0
lot/templates/new_annotation.html → lot/templates/new_property.html
View File

8
lot/templates/annotations.html → lot/templates/properties.html
View File

@ -11,15 +11,15 @@
<div class="row">
<div class="tab-pane fade show active" id="details">
<div class="btn-group dropdown ml-1 mt-1" uib-dropdown="">
<a href="{% url 'lot:add_annotation' lot.pk %}" class="btn btn-primary">
<a href="{% url 'lot:add_property' lot.pk %}" class="btn btn-primary">
<i class="bi bi-plus"></i>
Add new annotation
Add new lot Property
<span class="caret"></span>
</a>
</div>
<h5 class="card-title mt-2">Annotations</h5>
<h5 class="card-title mt-2">Properties</h5>
<table class="table table-striped">
<thead>
<tr>
@ -31,7 +31,7 @@
</tr>
</thead>
<tbody>
{% for a in annotations %}
{% for a in properties %}
<tr>
<td>{{ a.key }}</td>
<td>{{ a.value }}</td>

4
lot/urls.py
View File

@ -12,6 +12,6 @@ urlpatterns = [
path("tag/<int:pk>/", views.LotsTagsView.as_view(), name="tag"),
path("<int:pk>/document/", views.LotDocumentsView.as_view(), name="documents"),
path("<int:pk>/document/add", views.LotAddDocumentView.as_view(), name="add_document"),
path("<int:pk>/annotation", views.LotAnnotationsView.as_view(), name="annotations"),
path("<int:pk>/annotation/add", views.LotAddAnnotationView.as_view(), name="add_annotation"),
path("<int:pk>/property", views.LotPropertiesView.as_view(), name="properties"),
path("<int:pk>/property/add", views.LotAddPropertyView.as_view(), name="add_property"),
]

44
lot/views.py
View File

@ -9,9 +9,9 @@ from django.views.generic.edit import (
FormView,
)
from dashboard.mixins import DashboardView
from lot.models import Lot, LotTag, LotAnnotation
from lot.models import Lot, LotTag, LotProperty
from lot.forms import LotsForm
from device.models import Property
class NewLotView(DashboardView, CreateView):
template_name = "new_lot.html"
@ -143,18 +143,18 @@ class LotsTagsView(DashboardView, TemplateView):
class LotAddDocumentView(DashboardView, CreateView):
template_name = "new_annotation.html"
template_name = "new_property.html"
title = _("New Document")
breadcrumb = "Device / New document"
success_url = reverse_lazy('dashboard:unassigned_devices')
model = LotAnnotation
model = LotProperty
fields = ("key", "value")
def form_valid(self, form):
form.instance.owner = self.request.user.institution
form.instance.user = self.request.user
form.instance.lot = self.lot
form.instance.type = LotAnnotation.Type.DOCUMENT
form.instance.type = Property.Type.DOCUMENT
response = super().form_valid(form)
return response
@ -169,16 +169,16 @@ class LotAddDocumentView(DashboardView, CreateView):
class LotDocumentsView(DashboardView, TemplateView):
template_name = "documents.html"
title = _("New Document")
breadcrumb = "Device / New document"
breadcrumb = "Devicce / New document"
def get_context_data(self, **kwargs):
self.pk = kwargs.get('pk')
context = super().get_context_data(**kwargs)
lot = get_object_or_404(Lot, owner=self.request.user.institution, id=self.pk)
documents = LotAnnotation.objects.filter(
documents = LotProperty.objects.filter(
lot=lot,
owner=self.request.user.institution,
type=LotAnnotation.Type.DOCUMENT,
type=Property.Type.DOCUMENT,
)
context.update({
'lot': lot,
@ -189,48 +189,48 @@ class LotDocumentsView(DashboardView, TemplateView):
return context
class LotAnnotationsView(DashboardView, TemplateView):
template_name = "annotations.html"
title = _("New Annotation")
breadcrumb = "Device / New annotation"
class LotPropertiesView(DashboardView, TemplateView):
template_name = "properties.html"
title = _("New Lot Property")
breadcrumb = "Lot / New property"
def get_context_data(self, **kwargs):
self.pk = kwargs.get('pk')
context = super().get_context_data(**kwargs)
lot = get_object_or_404(Lot, owner=self.request.user.institution, id=self.pk)
annotations = LotAnnotation.objects.filter(
properties = LotProperty.objects.filter(
lot=lot,
owner=self.request.user.institution,
type=LotAnnotation.Type.USER,
type=Property.Type.USER,
)
context.update({
'lot': lot,
'annotations': annotations,
'properties': properties,
'title': self.title,
'breadcrumb': self.breadcrumb
})
return context
class LotAddAnnotationView(DashboardView, CreateView):
template_name = "new_annotation.html"
title = _("New Annotation")
breadcrumb = "Device / New annotation"
class LotAddPropertyView(DashboardView, CreateView):
template_name = "new_property.html"
title = _("New Lot Property")
breadcrumb = "Device / New property"
success_url = reverse_lazy('dashboard:unassigned_devices')
model = LotAnnotation
model = LotProperty
fields = ("key", "value")
def form_valid(self, form):
form.instance.owner = self.request.user.institution
form.instance.user = self.request.user
form.instance.lot = self.lot
form.instance.type = LotAnnotation.Type.USER
form.instance.type = Property.Type.USER
response = super().form_valid(form)
return response
def get_form_kwargs(self):
pk = self.kwargs.get('pk')
self.lot = get_object_or_404(Lot, pk=pk, owner=self.request.user.institution)
self.success_url = reverse_lazy('lot:annotations', args=[pk])
self.success_url = reverse_lazy('lot:properties', args=[pk])
kwargs = super().get_form_kwargs()
return kwargs

20
utils/device.py
View File

@ -6,7 +6,7 @@ import logging
from django.core.exceptions import ValidationError
from evidence.xapian import index
from evidence.models import Annotation
from evidence.models import SystemProperty, Property
from device.models import Device
@ -68,7 +68,7 @@ def create_doc(data):
return doc
def create_annotation(doc, user, commit=False):
def create_property(doc, user, commit=False):
if not doc or not doc.get('uuid') or not doc.get("CUSTOMER_ID"):
return []
@ -76,25 +76,25 @@ def create_annotation(doc, user, commit=False):
'uuid': doc['uuid'],
'owner': user.institution,
'user': user,
'type': Annotation.Type.SYSTEM,
'type': Property.Type.SYSTEM,
'key': 'CUSTOMER_ID',
'value': doc['CUSTOMER_ID'],
}
if commit:
annotation = Annotation.objects.filter(
property = SystemProperty.objects.filter(
uuid=doc["uuid"],
owner=user.institution,
type=Annotation.Type.SYSTEM,
type=Property.Type.SYSTEM,
)
if annotation:
txt = "Warning: Snapshot %s already registered (annotation exists)"
if property:
txt = "Warning: Snapshot %s already registered (system property exists)"
logger.warning(txt, doc["uuid"])
return annotation
return property
return Annotation.objects.create(**data)
return SystemProperty.objects.create(**data)
return Annotation(**data)
return SystemProperty(**data)
def create_index(doc, user):