From d154617ca34fc574f0e2a1ca40c5e0555ae5eed6 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 8 Nov 2023 11:24:37 +0100 Subject: [PATCH] refactor form import csv --- idhub/admin/forms.py | 119 ++++++++++++++++++ idhub/admin/views.py | 110 ++++------------ idhub/templates/idhub/admin/import.html | 2 +- idhub/templates/idhub/admin/import_add.html | 44 ++++--- idhub/templates/idhub/admin/import_step2.html | 34 ----- idhub/templates/idhub/admin/import_step3.html | 32 ----- idhub/urls.py | 6 +- 7 files changed, 165 insertions(+), 182 deletions(-) delete mode 100644 idhub/templates/idhub/admin/import_step2.html delete mode 100644 idhub/templates/idhub/admin/import_step3.html diff --git a/idhub/admin/forms.py b/idhub/admin/forms.py index 22b4954..83c167c 100644 --- a/idhub/admin/forms.py +++ b/idhub/admin/forms.py @@ -1,9 +1,128 @@ +import csv +import json +import copy +import pandas as pd +from jsonschema import validate + from django import forms +from django.core.exceptions import ValidationError +from idhub.models import ( + DID, + File_datas, + Schemas, + VerificableCredential, +) +from idhub_auth.models import User class ImportForm(forms.Form): + did = forms.ChoiceField(choices=[]) + schema = forms.ChoiceField(choices=[]) file_import = forms.FileField() + def __init__(self, *args, **kwargs): + self._schema = None + self._did = None + self.rows = {} + self.user = kwargs.pop('user', None) + super().__init__(*args, **kwargs) + self.fields['did'].choices = [ + (x.did, x.label) for x in DID.objects.filter(user=self.user) + ] + self.fields['schema'].choices = [ + (x.id, x.name()) for x in Schemas.objects.filter() + ] + + def clean_did(self): + data = self.cleaned_data["did"] + did = DID.objects.filter( + user=self.user, + did=data + ) + + if not did.exists(): + raise ValidationError("Did is not valid!") + + self._did = did.first() + + return data + + def clean_schema(self): + data = self.cleaned_data["schema"] + schema = Schemas.objects.filter( + id=data + ) + if not schema.exists(): + raise ValidationError("Schema is not valid!") + + self._schema = schema.first() + + return data + + def clean_file_import(self): + data = self.cleaned_data["file_import"] + self.file_name = data.name + if File_datas.objects.filter(file_name=self.file_name, success=True).exists(): + raise ValidationError("This file already exists!") + + self.json_schema = json.loads(self._schema.data) + df = pd.read_csv (data, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL) + data_pd = df.fillna('').to_dict() + + if not data_pd: + self.exception("This file is empty!") + + for n in range(df.last_valid_index()+1): + row = {} + for k in data_pd.keys(): + row[k] = data_pd[k][n] + + user = self.validate_jsonld(n, row) + self.rows[user] = row + + return data + + def save(self, commit=True): + table = [] + for k, v in self.rows.items(): + table.append(self.create_credential(k, v)) + + if commit: + for cred in table: + cred.save() + File_datas.objects.create(file_name=self.file_name) + return table + + return + + def validate_jsonld(self, line, row): + try: + validate(instance=row, schema=self.json_schema) + except Exception as e: + msg = "line {}: {}".format(line+1, e) + self.exception(msg) + + user = User.objects.filter(email=row.get('email')) + if not user: + txt = _('The user not exist!') + msg = "line {}: {}".format(line+1, txt) + self.exception(msg) + + return user.first() + + def create_credential(self, user, row): + d = copy.copy(self.json_schema) + d['instance'] = row + return VerificableCredential( + verified=False, + user=user, + data=json.dumps(d) + ) + + def exception(self, msg): + File_datas.objects.create(file_name=self.file_name, success=False) + raise ValidationError(msg) + class SchemaForm(forms.Form): file_template = forms.FileField() diff --git a/idhub/admin/views.py b/idhub/admin/views.py index 3247308..0492f5d 100644 --- a/idhub/admin/views.py +++ b/idhub/admin/views.py @@ -1,5 +1,4 @@ import os -import csv import json import copy import logging @@ -11,7 +10,12 @@ from smtplib import SMTPException from django.conf import settings from django.utils.translation import gettext_lazy as _ from django.views.generic.base import TemplateView -from django.views.generic.edit import UpdateView, CreateView, DeleteView +from django.views.generic.edit import ( + CreateView, + DeleteView, + FormView, + UpdateView, +) from django.shortcuts import get_object_or_404, redirect from django.urls import reverse_lazy from django.http import HttpResponse @@ -61,7 +65,7 @@ class SchemasMix(AdminView, TemplateView): section = "Templates" -class ImportExport(AdminView, TemplateView): +class ImportExport(AdminView): title = _("Massive Data Management") section = "ImportExport" @@ -695,7 +699,7 @@ class SchemasImportAddView(SchemasMix): return data -class ImportView(ImportExport): +class ImportView(ImportExport, TemplateView): template_name = "idhub/admin/import.html" subtitle = _('Import') icon = '' @@ -708,7 +712,7 @@ class ImportView(ImportExport): return context -class ImportStep2View(ImportExport): +class ImportStep2View(ImportExport, TemplateView): template_name = "idhub/admin/import_step2.html" subtitle = _('Import') icon = '' @@ -721,93 +725,23 @@ class ImportStep2View(ImportExport): return context -class ImportStep3View(ImportExport): - template_name = "idhub/admin/import_step3.html" +class ImportAddView(ImportExport, FormView): + template_name = "idhub/admin/import_add.html" subtitle = _('Import') icon = '' + form_class = ImportForm success_url = reverse_lazy('idhub:admin_import') - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context.update({ - 'form': ImportForm(), - }) - return context + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs['user'] = self.request.user + return kwargs - def post(self, request, *args, **kwargs): - self.pk = kwargs['pk'] - self.schema = get_object_or_404(Schemas, pk=self.pk) - form = ImportForm(request.POST, request.FILES) - if form.is_valid(): - result = self.handle_uploaded_file() - if not result: - messages.error(request, _("There are some errors in the file")) - return super().get(request, *args, **kwargs) - return redirect(self.success_url) + def form_valid(self, form): + cred = form.save() + if cred: + messages.success(self.request, _("The file import was successfully!")) else: - return super().get(request, *args, **kwargs) - - return super().post(request, *args, **kwargs) - - def handle_uploaded_file(self): - f = self.request.FILES.get('file_import') - if not f: - messages.error(self.request, _("There aren't file")) - return - - file_name = f.name - if File_datas.objects.filter(file_name=file_name, success=True).exists(): - messages.error(self.request, _("This file already exists!")) - return - - self.json_schema = json.loads(self.schema.data) - df = pd.read_csv (f, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL) - data_pd = df.fillna('').to_dict() - rows = {} - - if not data_pd: - File_datas.objects.create(file_name=file_name, success=False) - return - - for n in range(df.last_valid_index()+1): - row = {} - for k in data_pd.keys(): - row[k] = data_pd[k][n] - - user = self.validate(n, row) - if not user: - File_datas.objects.create(file_name=file_name, success=False) - return - - rows[user] = row - - File_datas.objects.create(file_name=file_name) - for k, v in rows.items(): - self.create_credential(k, v) - - return True - - def validate(self, line, row): - try: - validate(instance=row, schema=self.json_schema) - except Exception as e: - messages.error(self.request, "line {}: {}".format(line+1, e)) - return - - user = User.objects.filter(email=row.get('email')) - if not user: - txt = _('The user not exist!') - messages.error(self.request, "line {}: {}".format(line+1, txt)) - return - - return user.first() - - def create_credential(self, user, row): - d = copy.copy(self.json_schema) - d['instance'] = row - return VerificableCredential.objects.create( - verified=False, - user=user, - data=json.dumps(d) - ) + messages.error(self.request, _("Error importing the file!")) + return super().form_valid(form) diff --git a/idhub/templates/idhub/admin/import.html b/idhub/templates/idhub/admin/import.html index a1b77be..0b74600 100644 --- a/idhub/templates/idhub/admin/import.html +++ b/idhub/templates/idhub/admin/import.html @@ -28,7 +28,7 @@
- {% translate "Import Datas" %} + {% translate "Import Datas" %}
diff --git a/idhub/templates/idhub/admin/import_add.html b/idhub/templates/idhub/admin/import_add.html index f18821f..7cb6270 100644 --- a/idhub/templates/idhub/admin/import_add.html +++ b/idhub/templates/idhub/admin/import_add.html @@ -6,29 +6,27 @@ {{ subtitle }} -
-
-
- - - - - - - - - - - {% for schema in schemas.all %} - - - - - - {% endfor %} - -
{{ schema.created_at }}{{ schema.file_schema }}{% trans 'Import Dates' %}
-
+{% load django_bootstrap5 %} +
+{% csrf_token %} +{% if form.errors %} + +{% endif %} +{% bootstrap_form form %} +
+ {% translate "Cancel" %} + +
+ +
{% endblock %} diff --git a/idhub/templates/idhub/admin/import_step2.html b/idhub/templates/idhub/admin/import_step2.html deleted file mode 100644 index f18821f..0000000 --- a/idhub/templates/idhub/admin/import_step2.html +++ /dev/null @@ -1,34 +0,0 @@ -{% extends "idhub/base_admin.html" %} -{% load i18n %} - -{% block content %} -

- - {{ subtitle }} -

-
-
-
- - - - - - - - - - - {% for schema in schemas.all %} - - - - - - {% endfor %} - -
{{ schema.created_at }}{{ schema.file_schema }}{% trans 'Import Dates' %}
-
-
-
-{% endblock %} diff --git a/idhub/templates/idhub/admin/import_step3.html b/idhub/templates/idhub/admin/import_step3.html deleted file mode 100644 index 7cb6270..0000000 --- a/idhub/templates/idhub/admin/import_step3.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "idhub/base_admin.html" %} -{% load i18n %} - -{% block content %} -

- - {{ subtitle }} -

-{% load django_bootstrap5 %} -
-{% csrf_token %} -{% if form.errors %} - -{% endif %} -{% bootstrap_form form %} -
- {% translate "Cancel" %} - -
- -
-{% endblock %} diff --git a/idhub/urls.py b/idhub/urls.py index 87ff563..785f4d1 100644 --- a/idhub/urls.py +++ b/idhub/urls.py @@ -169,8 +169,6 @@ urlpatterns = [ name='admin_schemas_import_add'), path('admin/import', views_admin.ImportView.as_view(), name='admin_import'), - path('admin/import/new', views_admin.ImportStep2View.as_view(), - name='admin_import_step2'), - path('admin/import//', views_admin.ImportStep3View.as_view(), - name='admin_import_step3'), + path('admin/import/new', views_admin.ImportAddView.as_view(), + name='admin_import_add'), ]