From 1bf2c1558d0b8418e8530069ab6f2a6f09349dca Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 14 Oct 2024 17:02:40 +0200 Subject: [PATCH 01/56] email system and reset password --- admin/views.py | 10 ++++++- dashboard/templates/unassigned_devices.html | 2 +- dhub/settings.py | 24 ++++++++++++++++ login/templates/activate_user_email.html | 31 +++++++++++++++++++++ login/templates/activate_user_email.txt | 19 +++++++++++++ login/templates/activate_user_subject.txt | 4 +++ login/templates/password_reset_email.html | 4 +-- login/templates/password_reset_email.txt | 2 +- login/views.py | 4 ++- 9 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 login/templates/activate_user_email.html create mode 100644 login/templates/activate_user_email.txt create mode 100644 login/templates/activate_user_subject.txt diff --git a/admin/views.py b/admin/views.py index 14627cb..9ce25c7 100644 --- a/admin/views.py +++ b/admin/views.py @@ -1,3 +1,4 @@ +from smtplib import SMTPException from django.urls import reverse_lazy from django.shortcuts import get_object_or_404 from django.utils.translation import gettext_lazy as _ @@ -9,6 +10,7 @@ from django.views.generic.edit import ( ) from dashboard.mixins import DashboardView, Http403 from user.models import User, Institution +from admin.email import NotifyActivateUserByEmail class AdminView(DashboardView): @@ -42,7 +44,7 @@ class UsersView(AdminView, TemplateView): return context -class CreateUserView(AdminView, CreateView): +class CreateUserView(AdminView, NotifyActivateUserByEmail, CreateView): template_name = "user.html" title = _("User") breadcrumb = _("admin / User") + " /" @@ -58,6 +60,12 @@ class CreateUserView(AdminView, CreateView): form.instance.institution = self.request.user.institution form.instance.set_password(form.instance.password) response = super().form_valid(form) + + try: + self.send_email(form.instance) + except SMTPException as e: + messages.error(self.request, e) + return response diff --git a/dashboard/templates/unassigned_devices.html b/dashboard/templates/unassigned_devices.html index af12820..369edd2 100644 --- a/dashboard/templates/unassigned_devices.html +++ b/dashboard/templates/unassigned_devices.html @@ -15,7 +15,7 @@ {% trans 'Documents' %} {% endif %} - + {% trans 'Exports' %} diff --git a/dhub/settings.py b/dhub/settings.py index 8b77fd7..3b48d2c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -39,6 +39,30 @@ assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) + +INITIAL_ADMIN_EMAIL = config("INITIAL_ADMIN_EMAIL", default='admin@example.org') +INITIAL_ADMIN_PASSWORD = config("INITIAL_ADMIN_PASSWORD", default='1234') + +DEFAULT_FROM_EMAIL = config( + 'DEFAULT_FROM_EMAIL', default='webmaster@localhost') + +EMAIL_HOST = config('EMAIL_HOST', default='localhost') + +EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='') + +EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='') + +EMAIL_PORT = config('EMAIL_PORT', default=25, cast=int) + +EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=False, cast=bool) + +EMAIL_BACKEND = config('EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend') + +EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') + +ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) + + # Application definition INSTALLED_APPS = [ diff --git a/login/templates/activate_user_email.html b/login/templates/activate_user_email.html new file mode 100644 index 0000000..20aa2a6 --- /dev/null +++ b/login/templates/activate_user_email.html @@ -0,0 +1,31 @@ +{% load i18n %}{% autoescape off %} +{% trans "DeviceHub" as site %} +

+ {% blocktrans %}You're receiving this email because your user account at {{site}} has been activated.{% endblocktrans %} +

+ +

+{% trans "Your username is:" %} {{ user.username }} +

+ +

+{% trans "Please go to the following page and choose a password:" %} +

+ +

+{% block reset_link %} + +{{ protocol }}://{{ domain }}{% url 'login:password_reset_confirm' uidb64=uid token=token %} + +{% endblock %} +

+ +

+{% trans "Thanks for using our site!" %} +

+ +

+{% blocktrans %}The {{site}} team{% endblocktrans %} +

+ +{% endautoescape %} diff --git a/login/templates/activate_user_email.txt b/login/templates/activate_user_email.txt new file mode 100644 index 0000000..2c4ff0c --- /dev/null +++ b/login/templates/activate_user_email.txt @@ -0,0 +1,19 @@ +{% load i18n %}{% autoescape off %} + +{% trans "DeviceHub" as site %} + +{% blocktrans %}You're receiving this email because your user account at {{site}} has been activated.{% endblocktrans %} + +{% trans "Your username is:" %} {{ user.username }} + +{% trans "Please go to the following page and choose a password:" %} +{% block reset_link %} +{{ protocol }}://{{ domain }}{% url 'login:password_reset_confirm' uidb64=uid token=token %} +{% endblock %} + + +{% trans "Thanks for using our site!" %} + +{% blocktrans %}The {{site}} team{% endblocktrans %} + +{% endautoescape %} diff --git a/login/templates/activate_user_subject.txt b/login/templates/activate_user_subject.txt new file mode 100644 index 0000000..d28a515 --- /dev/null +++ b/login/templates/activate_user_subject.txt @@ -0,0 +1,4 @@ +{% load i18n %}{% autoescape off %} +{% trans "IdHub" as site %} +{% blocktrans %}User activation on {{site}}{% endblocktrans %} +{% endautoescape %} diff --git a/login/templates/password_reset_email.html b/login/templates/password_reset_email.html index 062025a..903cc2e 100644 --- a/login/templates/password_reset_email.html +++ b/login/templates/password_reset_email.html @@ -9,8 +9,8 @@

{% block reset_link %} - -{{ protocol }}://{{ domain }}{% url 'idhub:password_reset_confirm' uidb64=uid token=token %} + +{{ protocol }}://{{ domain }}{% url 'login:password_reset_confirm' uidb64=uid token=token %} {% endblock %}

diff --git a/login/templates/password_reset_email.txt b/login/templates/password_reset_email.txt index a4313fe..8dbf945 100644 --- a/login/templates/password_reset_email.txt +++ b/login/templates/password_reset_email.txt @@ -3,7 +3,7 @@ {% trans "Please go to the following page and choose a new password:" %} {% block reset_link %} -{{ protocol }}://{{ domain }}{% url 'idhub:password_reset_confirm' uidb64=uid token=token %} +{{ protocol }}://{{ domain }}{% url 'login:password_reset_confirm' uidb64=uid token=token %} {% endblock %} {% trans "Your username, in case you've forgotten:" %} {{ user.username }} diff --git a/login/views.py b/login/views.py index 7cd9156..5bf48a3 100644 --- a/login/views.py +++ b/login/views.py @@ -65,8 +65,10 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): + import pdb; pdb.set_trace() try: - return super().form_valid(form) + response = super().form_valid(form) + return response except Exception as err: logger.error(err) return HttpResponseRedirect(self.success_url) From 2b4010418060b124af8e50bc75df9969a1ea8911 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 14 Oct 2024 17:43:32 +0200 Subject: [PATCH 02/56] change response from api/snapshots --- api/views.py | 23 ++++++++++++++++++++++- device/models.py | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index cfb7d6e..62477ff 100644 --- a/api/views.py +++ b/api/views.py @@ -1,5 +1,6 @@ import json +from django.conf import settings from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ @@ -69,9 +70,29 @@ def NewSnapshot(request): except Exception: return JsonResponse({'status': 'fail'}, status=200) - return JsonResponse({'status': 'success'}, status=200) + annotation = Annotation.objects.filter( + uuid=data['uuid'], + type=Annotation.Type.SYSTEM, + key="hidalgo1", + owner=tk.owner.institution + ).first() + if not annotation: + return JsonResponse({'status': 'fail'}, status=200) + + url = "{}://{}{}".format( + request.scheme, + settings.DOMAIN, + reverse_lazy("device:details", args=(annotation.value,)) + ) + response = { + "status": "success", + "dhid": annotation.value[:5].upper(), + "url": url, + "public_url": url + } + return JsonResponse(response, status=200) class TokenView(DashboardView, SingleTableView): diff --git a/device/models.py b/device/models.py index d225946..926e447 100644 --- a/device/models.py +++ b/device/models.py @@ -27,7 +27,7 @@ class Device: # the id is the chid of the device self.id = kwargs["id"] self.pk = self.id - self.shortid = self.pk[:6] + self.shortid = self.pk[:6].upper() self.algorithm = None self.owner = None self.annotations = [] From e47a7d80f2c1feb8ea9b6c981c63a3a4df4b5041 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 14 Oct 2024 17:54:58 +0200 Subject: [PATCH 03/56] fix dhid 6 characters --- api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 62477ff..72f5c12 100644 --- a/api/views.py +++ b/api/views.py @@ -88,7 +88,7 @@ def NewSnapshot(request): ) response = { "status": "success", - "dhid": annotation.value[:5].upper(), + "dhid": annotation.value[:6].upper(), "url": url, "public_url": url } From 4cad3eb0632436589ea8b004f97e87cd909121d4 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 14 Oct 2024 20:08:50 +0200 Subject: [PATCH 04/56] relevant note for current snapshot api --- api/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/views.py b/api/views.py index 72f5c12..175fe3e 100644 --- a/api/views.py +++ b/api/views.py @@ -73,6 +73,7 @@ def NewSnapshot(request): annotation = Annotation.objects.filter( uuid=data['uuid'], type=Annotation.Type.SYSTEM, + # TODO this is hardcoded, it should select the user preferred algorithm key="hidalgo1", owner=tk.owner.institution ).first() From b4c4ed2689b9ff74310a8b9d2d20fdda6a57d5c2 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:06:00 +0200 Subject: [PATCH 05/56] email up --- admin/email.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 admin/email.py diff --git a/admin/email.py b/admin/email.py new file mode 100644 index 0000000..1dea252 --- /dev/null +++ b/admin/email.py @@ -0,0 +1,69 @@ +import logging + +from django.conf import settings +from django.template import loader +from django.core.mail import EmailMultiAlternatives +from django.contrib.auth.tokens import default_token_generator +from django.contrib.sites.shortcuts import get_current_site +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode + + +logger = logging.getLogger(__name__) + + +class NotifyActivateUserByEmail: + subject_template_name = 'activate_user_subject.txt' + email_template_name = 'activate_user_email.txt' + html_email_template_name = 'activate_user_email.html' + + def get_email_context(self, user, token): + """ + Define a new context with a token for put in a email + when send a email for add a new password + """ + protocol = 'https' if self.request.is_secure() else 'http' + current_site = get_current_site(self.request) + site_name = current_site.name + domain = current_site.domain + if not token: + token = default_token_generator.make_token(user) + + context = { + 'email': user.email, + 'domain': domain, + 'site_name': site_name, + 'uid': urlsafe_base64_encode(force_bytes(user.pk)), + 'user': user, + 'token': token, + 'protocol': protocol, + } + return context + + def send_email(self, user, token=None): + """ + Send a email when a user is activated. + """ + context = self.get_email_context(user, token) + subject = loader.render_to_string(self.subject_template_name, context) + # Email subject *must not* contain newlines + subject = ''.join(subject.splitlines()) + body = loader.render_to_string(self.email_template_name, context) + from_email = settings.DEFAULT_FROM_EMAIL + to_email = user.email + + email_message = EmailMultiAlternatives( + subject, body, from_email, [to_email]) + html_email = loader.render_to_string(self.html_email_template_name, context) + email_message.attach_alternative(html_email, 'text/html') + try: + if settings.ENABLE_EMAIL: + email_message.send() + return + + logger.warning(to_email) + logger.warning(body) + + except Exception as err: + logger.error(err) + return From d04216ad795f0c8d84c542b7787621c1dab54fa0 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 15 Oct 2024 13:09:34 +0200 Subject: [PATCH 06/56] api snapshot public_url bugfix with the following code the URL is more appropriate (it basically mirrors the user's request) before: http://localhost/device/{shortid} after http://localhost:8000/device/{shortid} --- api/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index 175fe3e..fe68079 100644 --- a/api/views.py +++ b/api/views.py @@ -1,6 +1,5 @@ import json -from django.conf import settings from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ @@ -82,15 +81,14 @@ def NewSnapshot(request): if not annotation: return JsonResponse({'status': 'fail'}, status=200) - url = "{}://{}{}".format( - request.scheme, - settings.DOMAIN, - reverse_lazy("device:details", args=(annotation.value,)) - ) + url_args = reverse_lazy("device:details", args=(annotation.value,)) + url = request.build_absolute_uri(url_args) + response = { "status": "success", "dhid": annotation.value[:6].upper(), "url": url, + # TODO replace with public_url when available "public_url": url } return JsonResponse(response, status=200) From 92d9837a9172976ce0ccff24b3653644a8dd56d9 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:05:43 +0200 Subject: [PATCH 07/56] save snapshots on disk from api --- api/views.py | 42 +++++++++++++++++++++++++++++++++++++++--- dhub/settings.py | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index 175fe3e..9e3c0ad 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,8 @@ +import os import json +import shutil + +from datetime import datetime from django.conf import settings from django.urls import reverse_lazy @@ -24,8 +28,37 @@ from api.models import Token from api.tables import TokensTable +def move_json(path_name, user): + tmp_snapshots = settings.SNAPSHOTS_DIR + path_dir = os.path.join(tmp_snapshots, user) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + def save_in_disk(data, user): - pass + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.SNAPSHOTS_DIR + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name + @csrf_exempt @@ -60,10 +93,11 @@ def NewSnapshot(request): ).first() if exist_annotation: - raise ValidationError("error: the snapshot {} exist".format(data['uuid'])) + txt = "error: the snapshot {} exist".format(data['uuid']) + return JsonResponse({'status': txt}, status=500) # Process snapshot - # save_in_disk(data, tk.user) + path_name = save_in_disk(data, tk.owner.institution.name) try: Build(data, tk.owner) @@ -93,6 +127,8 @@ def NewSnapshot(request): "url": url, "public_url": url } + move_json(path_name, tk.owner.institution.name) + return JsonResponse(response, status=200) diff --git a/dhub/settings.py b/dhub/settings.py index 3b48d2c..3a55985 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -62,6 +62,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) +SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") # Application definition From 5bf5bc23600dff3fe5ce0ba6786731ea501d79ac Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 16:58:51 +0200 Subject: [PATCH 08/56] save placeholders --- api/views.py | 41 +++-------------------------------------- device/forms.py | 6 +++++- dhub/settings.py | 3 ++- evidence/forms.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/api/views.py b/api/views.py index 9e3c0ad..cf80b35 100644 --- a/api/views.py +++ b/api/views.py @@ -1,11 +1,10 @@ -import os import json -import shutil -from datetime import datetime +from uuid import uuid4 from django.conf import settings from django.urls import reverse_lazy +from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt @@ -17,9 +16,8 @@ from django.views.generic.edit import ( DeleteView, UpdateView, ) -from django.http import JsonResponse -from uuid import uuid4 +from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -28,39 +26,6 @@ from api.models import Token from api.tables import TokensTable -def move_json(path_name, user): - tmp_snapshots = settings.SNAPSHOTS_DIR - path_dir = os.path.join(tmp_snapshots, user) - - if os.path.isfile(path_name): - shutil.copy(path_name, path_dir) - os.remove(path_name) - - -def save_in_disk(data, user): - uuid = data.get('uuid', '') - now = datetime.now() - year = now.year - month = now.month - day = now.day - hour = now.hour - minutes = now.minute - tmp_snapshots = settings.SNAPSHOTS_DIR - - name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" - path_dir = os.path.join(tmp_snapshots, user, "errors") - path_name = os.path.join(path_dir, name_file) - - if not os.path.isdir(path_dir): - os.system(f'mkdir -p {path_dir}') - - with open(path_name, 'w') as snapshot_file: - snapshot_file.write(json.dumps(data)) - - return path_name - - - @csrf_exempt def NewSnapshot(request): # Accept only posts diff --git a/device/forms.py b/device/forms.py index b2f0d3c..38d649e 100644 --- a/device/forms.py +++ b/device/forms.py @@ -1,5 +1,6 @@ from django import forms from utils.device import create_annotation, create_doc, create_index +from utils.save_snapshots import move_json, save_in_disk DEVICE_TYPES = [ @@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet): doc = create_doc(row) if not commit: return doc - + + path_name = save_in_disk(doc, self.user.name) create_index(doc, self.user) create_annotation(doc, user, commit=commit) + move_json(path_name, self.user.name) + return doc diff --git a/dhub/settings.py b/dhub/settings.py index 3a55985..c307073 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os import xapian from pathlib import Path @@ -62,7 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) -SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") +EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db")) # Application definition diff --git a/evidence/forms.py b/evidence/forms.py index db98f2d..a47024e 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -9,6 +9,7 @@ from utils.forms import MultipleFileField from device.models import Device from evidence.parse import Build from evidence.models import Annotation +from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): @@ -48,7 +49,9 @@ class UploadForm(forms.Form): return for ev in self.evidences: + path_name = save_in_disk(ev[1], user.institution.name) Build(ev[1], user) + move_json(path_name, user.institution.name) class UserTagForm(forms.Form): @@ -151,8 +154,10 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - cred.save() - create_index(doc, self.user) - return table + path_name = save_in_disk(doc, self.user.name) + cred.save() + create_index(doc, self.user) + move_json(path_name, self.user.name) + return table return From 69bd81289f86a18e55057d62f34c37518ef9ed41 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:56:34 +0200 Subject: [PATCH 09/56] add save snapshots for imports and uploads --- device/forms.py | 4 ++-- evidence/forms.py | 5 +++-- utils/save_snapshots.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 utils/save_snapshots.py diff --git a/device/forms.py b/device/forms.py index 38d649e..6c4d6e2 100644 --- a/device/forms.py +++ b/device/forms.py @@ -57,10 +57,10 @@ class BaseDeviceFormSet(forms.BaseFormSet): if not commit: return doc - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") create_index(doc, self.user) create_annotation(doc, user, commit=commit) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return doc diff --git a/evidence/forms.py b/evidence/forms.py index a47024e..cf90a71 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -154,10 +154,11 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") + cred.save() create_index(doc, self.user) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return table return diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py new file mode 100644 index 0000000..8c02f06 --- /dev/null +++ b/utils/save_snapshots.py @@ -0,0 +1,43 @@ +import os +import json +import shutil + +from datetime import datetime +from django.conf import settings + + +def move_json(path_name, user, place="snapshots"): + if place != "snapshots": + place = "placeholders" + + tmp_snapshots = settings.EVIDENCES_DIR + path_dir = os.path.join(tmp_snapshots, user, place) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + +def save_in_disk(data, user, place="snapshots"): + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.EVIDENCES_DIR + if place != "snapshots": + place = "placeholders" + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, place, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name From d1cd44f8f66f864b134053e96aeedf4383d4e05f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:57:45 +0200 Subject: [PATCH 10/56] remove pdbs --- login/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/login/views.py b/login/views.py index 5bf48a3..be6b009 100644 --- a/login/views.py +++ b/login/views.py @@ -65,7 +65,6 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): - import pdb; pdb.set_trace() try: response = super().form_valid(form) return response From 2d7b74c556efd5dd18ab82f685de5fab78c21131 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:17:35 +0200 Subject: [PATCH 11/56] fix bug of tag --- dashboard/views.py | 12 ++---- device/models.py | 102 +++++++++++++++++++++++++++++++++++++++++---- evidence/models.py | 22 ++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index b34e2a6..f056523 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -69,23 +69,17 @@ class SearchView(InventaryMixin): if not matches.size(): return self.search_hids(query, offset, limit) - annotations = [] + devices = [] for x in matches: - annotations.extend(self.get_annotations(x)) + devices.append(self.get_annotations(x)) - devices = [Device(id=x) for x in set(annotations)] count = matches.size() return devices, count def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - - return Annotation.objects.filter( - type=Annotation.Type.SYSTEM, - owner=self.request.user.institution, - uuid=uuid - ).values_list("value", flat=True).distinct() + return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): qry = Q() diff --git a/device/models.py b/device/models.py index 926e447..6169f2e 100644 --- a/device/models.py +++ b/device/models.py @@ -118,9 +118,32 @@ class Device: def get_unassigned(cls, institution, offset=0, limit=None): sql = """ - SELECT DISTINCT t1.value from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type} + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, @@ -144,18 +167,83 @@ class Device: def get_unassigned_count(cls, institution): sql = """ - SELECT count(DISTINCT t1.value) from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type}; + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT + COUNT(DISTINCT value) + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, ) - with connection.cursor() as cursor: cursor.execute(sql) return cursor.fetchall()[0][0] + @classmethod + def get_annotation_from_uuid(cls, uuid, institution): + sql = """ + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + AND t1.uuid = '{uuid}' + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1; + """.format( + uuid=uuid.replace("-", ""), + institution=institution.id, + type=Annotation.Type.SYSTEM, + ) + + annotations = [] + with connection.cursor() as cursor: + cursor.execute(sql) + annotations = cursor.fetchall() + + return cls(id=annotations[0][0]) + @property def is_websnapshot(self): if not self.last_evidence: diff --git a/evidence/models.py b/evidence/models.py index 667945f..ffc0d84 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,8 +126,30 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, + key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() + # annotations = Annotation.objects.filter( + # owner=user.institution, + # type=Annotation.Type.SYSTEM + # ) + + # annotations = annotations.annotate( + # priority=models.Case( + # models.When(key="CUSTOM_ID", then=1), + # models.When(key="hidalgo1", then=2), + # default=3, + # output_field=models.IntegerField(), + # ) + # ) + + # # sqlite not soport distinct('uuid') + # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # # 'uuid').values_list("uuid", flat=True) + # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # ).values_list("uuid", flat=True) + # return set(filtered_uuids) + def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From ccda270de086ee4d87769e4efcd3a9fb1101d586 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:19:09 +0200 Subject: [PATCH 12/56] clean code comment --- evidence/models.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index ffc0d84..354cb89 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -129,27 +129,6 @@ class Evidence: key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() - # annotations = Annotation.objects.filter( - # owner=user.institution, - # type=Annotation.Type.SYSTEM - # ) - - # annotations = annotations.annotate( - # priority=models.Case( - # models.When(key="CUSTOM_ID", then=1), - # models.When(key="hidalgo1", then=2), - # default=3, - # output_field=models.IntegerField(), - # ) - # ) - - # # sqlite not soport distinct('uuid') - # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # # 'uuid').values_list("uuid", flat=True) - # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # ).values_list("uuid", flat=True) - # return set(filtered_uuids) - def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From cd3cc62004bd5dc36ba75004bce07f3611896fbb Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 17:40:00 +0200 Subject: [PATCH 13/56] fix hid in details of device --- device/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/device/models.py b/device/models.py index 6169f2e..1f43476 100644 --- a/device/models.py +++ b/device/models.py @@ -90,9 +90,11 @@ class Device: def get_hids(self): annotations = self.get_annotations() + algos = list(ALGOS.keys()) + algos.append('CUSTOM_ID') self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, - key__in=ALGOS.keys(), + key__in=algos, ).values_list("value", flat=True))) def get_evidences(self): From 94fd1d685a5e31d216c9a78344dbac164d035c21 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 20:39:57 +0200 Subject: [PATCH 14/56] docker-reset: remove better the db now that in db dir there are snapshots stored --- docker-reset.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index cdbb971..bf14db2 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,13 @@ set -u set -x main() { + cd "$(dirname "${0}")" + if [ "${DETACH:-}" ]; then detach_arg='-d' fi # remove old database - sudo rm -vf db/* + sudo rm -vfr ./db/* docker compose down -v docker compose build docker compose up ${detach_arg:-} From 7b95ce054119fd668879bd099f002fa201fb4f84 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:09:16 +0200 Subject: [PATCH 15/56] docker: make default user admin --- docker/devicehub-django.entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 920cdf0..9361fdb 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,9 +24,10 @@ deploy() { INIT_ORG="${INIT_ORG:-example-org}" INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" + ADMIN='True' ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" From 11f4b647c219a768e96e79474027b83085f52b84 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:14:22 +0200 Subject: [PATCH 16/56] snapshot processing: improve warning msg add comment TODO that system annotation should happen --- evidence/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..dfc5a1f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -113,7 +113,8 @@ class Build: # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" if not mac: - print("WARNING!! No there are MAC address") + print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" ) + # TODO generate system annotation for that snapshot else: print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}") From 71f68e2d4b1f4588d22f28ce42cc46d9e0afb358 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:54:54 +0200 Subject: [PATCH 17/56] add predefined_token as an alternative to the randomly generated, which is also possible when no predefined_token is defined also update .env.example vars --- .env.example | 19 +++++++++++++++++++ docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 ++- user/management/commands/add_user.py | 12 +++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index cf39121..1495f9d 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,21 @@ DOMAIN=localhost DEMO=false + +STATIC_ROOT=/tmp/static/ +MEDIA_ROOT=/tmp/media/ +ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DOMAIN=localhost +DEBUG=True +EMAIL_HOST="mail.example.org" +EMAIL_HOST_USER="fillme_noreply" +EMAIL_HOST_PASSWORD="fillme_passwd" +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" +EMAIL_FILE_PATH="/tmp/app-messages" +ENABLE_EMAIL=false +COMMIT=$(git log --format="%H %ad" --date=iso -n 1) +PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' +# TODO review these vars +#SNAPSHOTS_DIR=/path/to/TODO +#EVIDENCES_DIR=/path/to/TODO diff --git a/docker-compose.yml b/docker-compose.yml index ee59640..7d144bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - DEMO=${DEMO:-n} + - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 9361fdb..e988d4c 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -25,9 +25,10 @@ deploy() { INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" ADMIN='True' + PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 60f3dd2..6593fee 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -17,15 +17,17 @@ class Command(BaseCommand): parser.add_argument('email', type=str, help='email') parser.add_argument('password', type=str, help='password') parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin') + parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token') def handle(self, *args, **kwargs): email = kwargs['email'] password = kwargs['password'] is_admin = kwargs['is_admin'] + predefined_token = kwargs['predefined_token'] institution = Institution.objects.get(name=kwargs['institution']) - self.create_user(institution, email, password, is_admin) + self.create_user(institution, email, password, is_admin, predefined_token) - def create_user(self, institution, email, password, is_admin): + def create_user(self, institution, email, password, is_admin, predefined_token): self.u = User.objects.create( institution=institution, email=email, @@ -34,6 +36,10 @@ class Command(BaseCommand): ) self.u.set_password(password) self.u.save() - token = uuid4() + if predefined_token: + token = predefined_token + else: + token = uuid4() + Token.objects.create(token=token, owner=self.u) print(f"TOKEN: {token}") From ce9a69d6f7f9f3cb8869ba9d8170bfc3572e0dcd Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:05:09 +0200 Subject: [PATCH 18/56] docker: bugfix wrong detection of demo --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e988d4c..2c7c77b 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" ]; then + if [ "${DEMO:-}" = 'y' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 1751dc98c2e76e8b39045a86f893dc10d1a4024a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:07:40 +0200 Subject: [PATCH 19/56] docker: use demo true/false instead of y/n --- docker-compose.yml | 2 +- docker/devicehub-django.entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d144bd..b451358 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - DEMO=${DEMO:-n} + - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 2c7c77b..452b32d 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" = 'y' ]; then + if [ "${DEMO:-}" = 'true' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 2e638e40a9756a75c1ea7742396e07635afb0a19 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:48:42 +0200 Subject: [PATCH 20/56] add ALLOWED_HOSTS in docker compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b451358..9ea039c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From d94b12d1a0cdd62cb6cfe523cfba4733b3c6c8b0 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 18:38:57 +0200 Subject: [PATCH 21/56] locale in settings --- dhub/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhub/settings.py b/dhub/settings.py index c307073..e109072 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -164,6 +164,11 @@ USE_I18N = True USE_TZ = True +USE_L10N = True +LANGUAGES = [ + ('es', 'Spanish'), + ('en', 'English'), +] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ From 1bb3b1263642c26c904082c8d2cba41922a20f49 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 19:08:00 +0200 Subject: [PATCH 22/56] more debug in the error answer from api --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index cf80b35..5aa5f87 100644 --- a/api/views.py +++ b/api/views.py @@ -66,8 +66,8 @@ def NewSnapshot(request): try: Build(data, tk.owner) - except Exception: - return JsonResponse({'status': 'fail'}, status=200) + except Exception as err: + return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( uuid=data['uuid'], From 182f85457063c7022d7b7d73a5422ad9c373265e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 09:52:09 +0200 Subject: [PATCH 23/56] add commit_id in settings --- dashboard/mixins.py | 2 ++ dhub/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index aa63d36..af01a1b 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -1,4 +1,5 @@ from django.urls import resolve +from django.conf import settings from django.shortcuts import get_object_or_404, redirect, Http404 from django.utils.translation import gettext_lazy as _ from django.core.exceptions import PermissionDenied @@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ + "commit_id": settings.COMMIT, 'title': self.title, 'subtitle': self.subtitle, 'breadcrumb': self.breadcrumb, diff --git a/dhub/settings.py b/dhub/settings.py index e109072..b96710e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -213,3 +213,4 @@ LOGGING = { SNAPSHOT_PATH="/tmp/" DATA_UPLOAD_MAX_NUMBER_FILES = 1000 +COMMIT = config('COMMIT', default='') From 8e8130fc34f566cdd68be882d7e8255d94ba4763 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:20:23 +0200 Subject: [PATCH 24/56] docker: improve debug --- docker/devicehub-django.entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 452b32d..e5773f3 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + if [ "${DEBUG:-}" = 'true' ]; then + ./manage.py print_settings + fi + # detect if existing deployment (TODO only works with sqlite) if [ -f "${program_dir}/db/db.sqlite3" ]; then echo "INFO: detected EXISTING deployment" @@ -38,7 +42,7 @@ deploy() { runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" ]; then + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO From 460fb678e9722c6668102e61e2807cba23cc5a23 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:24:40 +0200 Subject: [PATCH 25/56] settings: improve assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index b96710e..80d9f3c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From b0754d8e97f4078f14dbd70cd31f37c1cfc186cd Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:26:38 +0200 Subject: [PATCH 26/56] settings: bugfix assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index 80d9f3c..c580e6b 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From a65d78e72f1fd74ec5eb679a63c0bcc07fd42ddf Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:27:27 +0200 Subject: [PATCH 27/56] settings: bugfix format assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index c580e6b..891b714 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 03b9c5fea8fa2bf1a79ac61ba539a323010cb546 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:28:02 +0200 Subject: [PATCH 28/56] docker-compose: bugfix DOMAIN vs ALLOWED_HOSTS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ea039c..d376313 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From f87897de85ccf64d8386968091a3d33f20e334c8 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:45:08 +0200 Subject: [PATCH 29/56] add commit footer for docker --- .env.example | 1 - docker/devicehub-django.entrypoint.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1495f9d..37ed087 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ EMAIL_USE_TLS=True EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" EMAIL_FILE_PATH="/tmp/app-messages" ENABLE_EMAIL=false -COMMIT=$(git log --format="%H %ad" --date=iso -n 1) PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' # TODO review these vars #SNAPSHOTS_DIR=/path/to/TODO diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e5773f3..bdf1008 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + # TODO this is weird, find better workaround + git config --global --add safe.directory /opt/devicehub-django + export COMMIT=$(git log --format="%H %ad" --date=iso -n 1) + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py print_settings fi From a08428c90b05aa222b81b1350ca899e093dbe5e4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 12:03:26 +0200 Subject: [PATCH 30/56] commit in login page --- login/templates/login.html | 1 + login/views.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/login/templates/login.html b/login/templates/login.html index 97ba4cb..a82882e 100644 --- a/login/templates/login.html +++ b/login/templates/login.html @@ -42,4 +42,5 @@ + {% endblock %} diff --git a/login/views.py b/login/views.py index be6b009..beb0368 100644 --- a/login/views.py +++ b/login/views.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.urls import reverse_lazy from django.contrib.auth import views as auth_views from django.contrib.auth import login as auth_login @@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView): extra_context = { 'title': _('Login'), 'success_url': reverse_lazy('dashboard:unassigned_devices'), - # 'commit_id': settings.COMMIT, + 'commit_id': settings.COMMIT, } def get(self, request, *args, **kwargs): From 69a54b92fb5318aab85f9e4429d3b9ad832cdb84 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:05:43 +0200 Subject: [PATCH 31/56] save snapshots on disk from api --- api/views.py | 42 +++++++++++++++++++++++++++++++++++++++--- dhub/settings.py | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index fe68079..0ae7e04 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,8 @@ +import os import json +import shutil + +from datetime import datetime from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect @@ -23,8 +27,37 @@ from api.models import Token from api.tables import TokensTable +def move_json(path_name, user): + tmp_snapshots = settings.SNAPSHOTS_DIR + path_dir = os.path.join(tmp_snapshots, user) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + def save_in_disk(data, user): - pass + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.SNAPSHOTS_DIR + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name + @csrf_exempt @@ -59,10 +92,11 @@ def NewSnapshot(request): ).first() if exist_annotation: - raise ValidationError("error: the snapshot {} exist".format(data['uuid'])) + txt = "error: the snapshot {} exist".format(data['uuid']) + return JsonResponse({'status': txt}, status=500) # Process snapshot - # save_in_disk(data, tk.user) + path_name = save_in_disk(data, tk.owner.institution.name) try: Build(data, tk.owner) @@ -91,6 +125,8 @@ def NewSnapshot(request): # TODO replace with public_url when available "public_url": url } + move_json(path_name, tk.owner.institution.name) + return JsonResponse(response, status=200) diff --git a/dhub/settings.py b/dhub/settings.py index 3b48d2c..3a55985 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -62,6 +62,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) +SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") # Application definition From fae269eb8d415f05e3c94cba446a4d629d85fd92 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 16:58:51 +0200 Subject: [PATCH 32/56] save placeholders --- api/views.py | 41 +++-------------------------------------- device/forms.py | 6 +++++- dhub/settings.py | 3 ++- evidence/forms.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/api/views.py b/api/views.py index 0ae7e04..72b5518 100644 --- a/api/views.py +++ b/api/views.py @@ -1,10 +1,9 @@ -import os import json -import shutil -from datetime import datetime +from uuid import uuid4 from django.urls import reverse_lazy +from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt @@ -16,9 +15,8 @@ from django.views.generic.edit import ( DeleteView, UpdateView, ) -from django.http import JsonResponse -from uuid import uuid4 +from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -27,39 +25,6 @@ from api.models import Token from api.tables import TokensTable -def move_json(path_name, user): - tmp_snapshots = settings.SNAPSHOTS_DIR - path_dir = os.path.join(tmp_snapshots, user) - - if os.path.isfile(path_name): - shutil.copy(path_name, path_dir) - os.remove(path_name) - - -def save_in_disk(data, user): - uuid = data.get('uuid', '') - now = datetime.now() - year = now.year - month = now.month - day = now.day - hour = now.hour - minutes = now.minute - tmp_snapshots = settings.SNAPSHOTS_DIR - - name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" - path_dir = os.path.join(tmp_snapshots, user, "errors") - path_name = os.path.join(path_dir, name_file) - - if not os.path.isdir(path_dir): - os.system(f'mkdir -p {path_dir}') - - with open(path_name, 'w') as snapshot_file: - snapshot_file.write(json.dumps(data)) - - return path_name - - - @csrf_exempt def NewSnapshot(request): # Accept only posts diff --git a/device/forms.py b/device/forms.py index b2f0d3c..38d649e 100644 --- a/device/forms.py +++ b/device/forms.py @@ -1,5 +1,6 @@ from django import forms from utils.device import create_annotation, create_doc, create_index +from utils.save_snapshots import move_json, save_in_disk DEVICE_TYPES = [ @@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet): doc = create_doc(row) if not commit: return doc - + + path_name = save_in_disk(doc, self.user.name) create_index(doc, self.user) create_annotation(doc, user, commit=commit) + move_json(path_name, self.user.name) + return doc diff --git a/dhub/settings.py b/dhub/settings.py index 3a55985..c307073 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os import xapian from pathlib import Path @@ -62,7 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) -SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") +EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db")) # Application definition diff --git a/evidence/forms.py b/evidence/forms.py index db98f2d..a47024e 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -9,6 +9,7 @@ from utils.forms import MultipleFileField from device.models import Device from evidence.parse import Build from evidence.models import Annotation +from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): @@ -48,7 +49,9 @@ class UploadForm(forms.Form): return for ev in self.evidences: + path_name = save_in_disk(ev[1], user.institution.name) Build(ev[1], user) + move_json(path_name, user.institution.name) class UserTagForm(forms.Form): @@ -151,8 +154,10 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - cred.save() - create_index(doc, self.user) - return table + path_name = save_in_disk(doc, self.user.name) + cred.save() + create_index(doc, self.user) + move_json(path_name, self.user.name) + return table return From 003d224c3e2c7f499f1ecd401ba475f0b3cefe02 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:56:34 +0200 Subject: [PATCH 33/56] add save snapshots for imports and uploads --- device/forms.py | 4 ++-- evidence/forms.py | 5 +++-- utils/save_snapshots.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 utils/save_snapshots.py diff --git a/device/forms.py b/device/forms.py index 38d649e..6c4d6e2 100644 --- a/device/forms.py +++ b/device/forms.py @@ -57,10 +57,10 @@ class BaseDeviceFormSet(forms.BaseFormSet): if not commit: return doc - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") create_index(doc, self.user) create_annotation(doc, user, commit=commit) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return doc diff --git a/evidence/forms.py b/evidence/forms.py index a47024e..cf90a71 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -154,10 +154,11 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") + cred.save() create_index(doc, self.user) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return table return diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py new file mode 100644 index 0000000..8c02f06 --- /dev/null +++ b/utils/save_snapshots.py @@ -0,0 +1,43 @@ +import os +import json +import shutil + +from datetime import datetime +from django.conf import settings + + +def move_json(path_name, user, place="snapshots"): + if place != "snapshots": + place = "placeholders" + + tmp_snapshots = settings.EVIDENCES_DIR + path_dir = os.path.join(tmp_snapshots, user, place) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + +def save_in_disk(data, user, place="snapshots"): + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.EVIDENCES_DIR + if place != "snapshots": + place = "placeholders" + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, place, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name From 2f9b61667d38dd84a2c0e4dd9dc5568a0054ba31 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:57:45 +0200 Subject: [PATCH 34/56] remove pdbs --- login/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/login/views.py b/login/views.py index 5bf48a3..be6b009 100644 --- a/login/views.py +++ b/login/views.py @@ -65,7 +65,6 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): - import pdb; pdb.set_trace() try: response = super().form_valid(form) return response From 090ecc275fbf6b7fc876268db4492b365f8dc4c3 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:17:35 +0200 Subject: [PATCH 35/56] fix bug of tag --- dashboard/views.py | 12 ++---- device/models.py | 102 +++++++++++++++++++++++++++++++++++++++++---- evidence/models.py | 22 ++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index b34e2a6..f056523 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -69,23 +69,17 @@ class SearchView(InventaryMixin): if not matches.size(): return self.search_hids(query, offset, limit) - annotations = [] + devices = [] for x in matches: - annotations.extend(self.get_annotations(x)) + devices.append(self.get_annotations(x)) - devices = [Device(id=x) for x in set(annotations)] count = matches.size() return devices, count def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - - return Annotation.objects.filter( - type=Annotation.Type.SYSTEM, - owner=self.request.user.institution, - uuid=uuid - ).values_list("value", flat=True).distinct() + return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): qry = Q() diff --git a/device/models.py b/device/models.py index 926e447..6169f2e 100644 --- a/device/models.py +++ b/device/models.py @@ -118,9 +118,32 @@ class Device: def get_unassigned(cls, institution, offset=0, limit=None): sql = """ - SELECT DISTINCT t1.value from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type} + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, @@ -144,18 +167,83 @@ class Device: def get_unassigned_count(cls, institution): sql = """ - SELECT count(DISTINCT t1.value) from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type}; + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT + COUNT(DISTINCT value) + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, ) - with connection.cursor() as cursor: cursor.execute(sql) return cursor.fetchall()[0][0] + @classmethod + def get_annotation_from_uuid(cls, uuid, institution): + sql = """ + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + AND t1.uuid = '{uuid}' + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1; + """.format( + uuid=uuid.replace("-", ""), + institution=institution.id, + type=Annotation.Type.SYSTEM, + ) + + annotations = [] + with connection.cursor() as cursor: + cursor.execute(sql) + annotations = cursor.fetchall() + + return cls(id=annotations[0][0]) + @property def is_websnapshot(self): if not self.last_evidence: diff --git a/evidence/models.py b/evidence/models.py index 667945f..ffc0d84 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,8 +126,30 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, + key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() + # annotations = Annotation.objects.filter( + # owner=user.institution, + # type=Annotation.Type.SYSTEM + # ) + + # annotations = annotations.annotate( + # priority=models.Case( + # models.When(key="CUSTOM_ID", then=1), + # models.When(key="hidalgo1", then=2), + # default=3, + # output_field=models.IntegerField(), + # ) + # ) + + # # sqlite not soport distinct('uuid') + # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # # 'uuid').values_list("uuid", flat=True) + # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # ).values_list("uuid", flat=True) + # return set(filtered_uuids) + def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From c422eaddebb9e6d6bfab9064ff087c2614bc0c7e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:19:09 +0200 Subject: [PATCH 36/56] clean code comment --- evidence/models.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index ffc0d84..354cb89 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -129,27 +129,6 @@ class Evidence: key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() - # annotations = Annotation.objects.filter( - # owner=user.institution, - # type=Annotation.Type.SYSTEM - # ) - - # annotations = annotations.annotate( - # priority=models.Case( - # models.When(key="CUSTOM_ID", then=1), - # models.When(key="hidalgo1", then=2), - # default=3, - # output_field=models.IntegerField(), - # ) - # ) - - # # sqlite not soport distinct('uuid') - # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # # 'uuid').values_list("uuid", flat=True) - # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # ).values_list("uuid", flat=True) - # return set(filtered_uuids) - def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From d8b6d3ded61df6f5c6d769f31cedaf13dd167730 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 17:40:00 +0200 Subject: [PATCH 37/56] fix hid in details of device --- device/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/device/models.py b/device/models.py index 6169f2e..1f43476 100644 --- a/device/models.py +++ b/device/models.py @@ -90,9 +90,11 @@ class Device: def get_hids(self): annotations = self.get_annotations() + algos = list(ALGOS.keys()) + algos.append('CUSTOM_ID') self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, - key__in=ALGOS.keys(), + key__in=algos, ).values_list("value", flat=True))) def get_evidences(self): From 3d10217599d14ce44e1d1b11fb83197e018e2b10 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 20:39:57 +0200 Subject: [PATCH 38/56] docker-reset: remove better the db now that in db dir there are snapshots stored --- docker-reset.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index cdbb971..bf14db2 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,13 @@ set -u set -x main() { + cd "$(dirname "${0}")" + if [ "${DETACH:-}" ]; then detach_arg='-d' fi # remove old database - sudo rm -vf db/* + sudo rm -vfr ./db/* docker compose down -v docker compose build docker compose up ${detach_arg:-} From 0e8607d93ecf45f3573220e97144e91a619c921a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:09:16 +0200 Subject: [PATCH 39/56] docker: make default user admin --- docker/devicehub-django.entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 920cdf0..9361fdb 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,9 +24,10 @@ deploy() { INIT_ORG="${INIT_ORG:-example-org}" INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" + ADMIN='True' ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" From f757c5835656b15c8daa32dee36e26c85d8009d5 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:14:22 +0200 Subject: [PATCH 40/56] snapshot processing: improve warning msg add comment TODO that system annotation should happen --- evidence/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..dfc5a1f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -113,7 +113,8 @@ class Build: # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" if not mac: - print("WARNING!! No there are MAC address") + print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" ) + # TODO generate system annotation for that snapshot else: print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}") From 67869bc6f55e45de170f8ae2222def8fe90d053f Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:54:54 +0200 Subject: [PATCH 41/56] add predefined_token as an alternative to the randomly generated, which is also possible when no predefined_token is defined also update .env.example vars --- .env.example | 19 +++++++++++++++++++ docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 ++- user/management/commands/add_user.py | 12 +++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index cf39121..1495f9d 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,21 @@ DOMAIN=localhost DEMO=false + +STATIC_ROOT=/tmp/static/ +MEDIA_ROOT=/tmp/media/ +ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DOMAIN=localhost +DEBUG=True +EMAIL_HOST="mail.example.org" +EMAIL_HOST_USER="fillme_noreply" +EMAIL_HOST_PASSWORD="fillme_passwd" +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" +EMAIL_FILE_PATH="/tmp/app-messages" +ENABLE_EMAIL=false +COMMIT=$(git log --format="%H %ad" --date=iso -n 1) +PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' +# TODO review these vars +#SNAPSHOTS_DIR=/path/to/TODO +#EVIDENCES_DIR=/path/to/TODO diff --git a/docker-compose.yml b/docker-compose.yml index ee59640..7d144bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - DEMO=${DEMO:-n} + - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 9361fdb..e988d4c 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -25,9 +25,10 @@ deploy() { INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" ADMIN='True' + PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 60f3dd2..6593fee 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -17,15 +17,17 @@ class Command(BaseCommand): parser.add_argument('email', type=str, help='email') parser.add_argument('password', type=str, help='password') parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin') + parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token') def handle(self, *args, **kwargs): email = kwargs['email'] password = kwargs['password'] is_admin = kwargs['is_admin'] + predefined_token = kwargs['predefined_token'] institution = Institution.objects.get(name=kwargs['institution']) - self.create_user(institution, email, password, is_admin) + self.create_user(institution, email, password, is_admin, predefined_token) - def create_user(self, institution, email, password, is_admin): + def create_user(self, institution, email, password, is_admin, predefined_token): self.u = User.objects.create( institution=institution, email=email, @@ -34,6 +36,10 @@ class Command(BaseCommand): ) self.u.set_password(password) self.u.save() - token = uuid4() + if predefined_token: + token = predefined_token + else: + token = uuid4() + Token.objects.create(token=token, owner=self.u) print(f"TOKEN: {token}") From bf47d4bc5d20a2ed61cfab04540494d37d789577 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:05:09 +0200 Subject: [PATCH 42/56] docker: bugfix wrong detection of demo --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e988d4c..2c7c77b 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" ]; then + if [ "${DEMO:-}" = 'y' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 425b032273bc4f6d77ff0f4e10df3bec0740cb80 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:07:40 +0200 Subject: [PATCH 43/56] docker: use demo true/false instead of y/n --- docker-compose.yml | 2 +- docker/devicehub-django.entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d144bd..b451358 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - DEMO=${DEMO:-n} + - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 2c7c77b..452b32d 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" = 'y' ]; then + if [ "${DEMO:-}" = 'true' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From c49644ffbd80990264c43e16d3d39c70bd82e1e1 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:48:42 +0200 Subject: [PATCH 44/56] add ALLOWED_HOSTS in docker compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b451358..9ea039c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 0b55d34d74ce092de75b9285d62b946b0942d809 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 18:38:57 +0200 Subject: [PATCH 45/56] locale in settings --- dhub/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhub/settings.py b/dhub/settings.py index c307073..e109072 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -164,6 +164,11 @@ USE_I18N = True USE_TZ = True +USE_L10N = True +LANGUAGES = [ + ('es', 'Spanish'), + ('en', 'English'), +] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ From e6141bcc7fabf528e82f812b895773a3f416972f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 19:08:00 +0200 Subject: [PATCH 46/56] more debug in the error answer from api --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 72b5518..79f3592 100644 --- a/api/views.py +++ b/api/views.py @@ -65,8 +65,8 @@ def NewSnapshot(request): try: Build(data, tk.owner) - except Exception: - return JsonResponse({'status': 'fail'}, status=200) + except Exception as err: + return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( uuid=data['uuid'], From b7c4926c390f8d91375e18bf7a655d55a992739d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 09:52:09 +0200 Subject: [PATCH 47/56] add commit_id in settings --- dashboard/mixins.py | 2 ++ dhub/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index aa63d36..af01a1b 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -1,4 +1,5 @@ from django.urls import resolve +from django.conf import settings from django.shortcuts import get_object_or_404, redirect, Http404 from django.utils.translation import gettext_lazy as _ from django.core.exceptions import PermissionDenied @@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ + "commit_id": settings.COMMIT, 'title': self.title, 'subtitle': self.subtitle, 'breadcrumb': self.breadcrumb, diff --git a/dhub/settings.py b/dhub/settings.py index e109072..b96710e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -213,3 +213,4 @@ LOGGING = { SNAPSHOT_PATH="/tmp/" DATA_UPLOAD_MAX_NUMBER_FILES = 1000 +COMMIT = config('COMMIT', default='') From 3d2cb7d184c381dc0e0cbdb9e97ca18015328f6a Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:20:23 +0200 Subject: [PATCH 48/56] docker: improve debug --- docker/devicehub-django.entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 452b32d..e5773f3 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + if [ "${DEBUG:-}" = 'true' ]; then + ./manage.py print_settings + fi + # detect if existing deployment (TODO only works with sqlite) if [ -f "${program_dir}/db/db.sqlite3" ]; then echo "INFO: detected EXISTING deployment" @@ -38,7 +42,7 @@ deploy() { runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" ]; then + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO From d995db8181361a7b8faf7466c252086baff55d02 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:24:40 +0200 Subject: [PATCH 49/56] settings: improve assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index b96710e..80d9f3c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 096b83da812ef66105d88382b0f69c19d0144ed6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:26:38 +0200 Subject: [PATCH 50/56] settings: bugfix assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index 80d9f3c..c580e6b 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 1d9d100fae1d490c55c2b89f3470c0908a4d23f6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:27:27 +0200 Subject: [PATCH 51/56] settings: bugfix format assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index c580e6b..891b714 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 9f05dee6a7656179e0ec396732ca651e61434a19 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:28:02 +0200 Subject: [PATCH 52/56] docker-compose: bugfix DOMAIN vs ALLOWED_HOSTS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ea039c..d376313 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 4cf164d4bce2656c3b388435ff8dafa7eae7fa8b Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:45:08 +0200 Subject: [PATCH 53/56] add commit footer for docker --- .env.example | 1 - docker/devicehub-django.entrypoint.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1495f9d..37ed087 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ EMAIL_USE_TLS=True EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" EMAIL_FILE_PATH="/tmp/app-messages" ENABLE_EMAIL=false -COMMIT=$(git log --format="%H %ad" --date=iso -n 1) PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' # TODO review these vars #SNAPSHOTS_DIR=/path/to/TODO diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e5773f3..bdf1008 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + # TODO this is weird, find better workaround + git config --global --add safe.directory /opt/devicehub-django + export COMMIT=$(git log --format="%H %ad" --date=iso -n 1) + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py print_settings fi From c60a9dd89d368e9dcf849a4dc470c7a57545d8e4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 12:03:26 +0200 Subject: [PATCH 54/56] commit in login page --- login/templates/login.html | 1 + login/views.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/login/templates/login.html b/login/templates/login.html index 97ba4cb..a82882e 100644 --- a/login/templates/login.html +++ b/login/templates/login.html @@ -42,4 +42,5 @@ + {% endblock %} diff --git a/login/views.py b/login/views.py index be6b009..beb0368 100644 --- a/login/views.py +++ b/login/views.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.urls import reverse_lazy from django.contrib.auth import views as auth_views from django.contrib.auth import login as auth_login @@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView): extra_context = { 'title': _('Login'), 'success_url': reverse_lazy('dashboard:unassigned_devices'), - # 'commit_id': settings.COMMIT, + 'commit_id': settings.COMMIT, } def get(self, request, *args, **kwargs): From d98e8e2d48597333e06bf781ac27588fd792f0ec Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 15:24:42 +0200 Subject: [PATCH 55/56] resolve conficts --- api/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 79f3592..69da2a2 100644 --- a/api/views.py +++ b/api/views.py @@ -7,9 +7,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt -from django.core.exceptions import ValidationError from django_tables2 import SingleTableView -from django.views.generic.base import View from django.views.generic.edit import ( CreateView, DeleteView, @@ -20,7 +18,6 @@ from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build -from user.models import User from api.models import Token from api.tables import TokensTable @@ -78,7 +75,7 @@ def NewSnapshot(request): if not annotation: - return JsonResponse({'status': 'fail'}, status=200) + return JsonResponse({'status': 'fail'}, status=500) url_args = reverse_lazy("device:details", args=(annotation.value,)) url = request.build_absolute_uri(url_args) From a0548b38c7b3445261f30cd1bc6b19e894083906 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Oct 2024 11:24:09 +0200 Subject: [PATCH 56/56] change EreuseWorkbench for workbench-script --- evidence/models.py | 10 +++++----- evidence/parse.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index 354cb89..eadb7a7 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -67,7 +67,7 @@ class Evidence: for xa in matches: self.doc = json.loads(xa.document.get_data()) - if self.doc.get("software") == "EreuseWorkbench": + if self.doc.get("software") == "workbench-script": dmidecode_raw = self.doc["data"]["dmidecode"] self.dmi = DMIParse(dmidecode_raw) @@ -80,7 +80,7 @@ class Evidence: self.created = self.annotations.last().created def get_components(self): - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc.get('components', []) self.set_components() return self.components @@ -92,7 +92,7 @@ class Evidence: return "" return list(self.doc.get('kv').values())[0] - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['manufacturer'] return self.dmi.manufacturer().strip() @@ -104,13 +104,13 @@ class Evidence: return "" return list(self.doc.get('kv').values())[1] - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['model'] return self.dmi.model().strip() def get_chassis(self): - if self.doc.get("software") != "EreuseWorkbench": + if self.doc.get("software") != "workbench-script": return self.doc['device']['model'] chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') diff --git a/evidence/parse.py b/evidence/parse.py index dfc5a1f..4f42934 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -57,7 +57,7 @@ class Build: } def get_hid_14(self): - if self.json.get("software") == "EreuseWorkbench": + if self.json.get("software") == "workbench-script": hid = self.get_hid(self.json) else: device = self.json['device']