refactor form import csv

idhub/admin/forms.py

@@ -1,9 +1,128 @@
+import csv
+import json
+import copy
+import pandas as pd
+from jsonschema import validate
+
 from django import forms
+from django.core.exceptions import ValidationError
+from idhub.models import (
+    DID,
+    File_datas, 
+    Schemas,
+    VerificableCredential,
+)
+from idhub_auth.models import User
 
 
 class ImportForm(forms.Form):
+    did = forms.ChoiceField(choices=[])
+    schema = forms.ChoiceField(choices=[])
     file_import = forms.FileField()
 
+    def __init__(self, *args, **kwargs):
+        self._schema = None
+        self._did = None
+        self.rows = {}
+        self.user = kwargs.pop('user', None)
+        super().__init__(*args, **kwargs)
+        self.fields['did'].choices = [
+            (x.did, x.label) for x in DID.objects.filter(user=self.user)
+        ]
+        self.fields['schema'].choices = [
+            (x.id, x.name()) for x in Schemas.objects.filter()
+        ]
+
+    def clean_did(self):
+        data = self.cleaned_data["did"]
+        did = DID.objects.filter(
+            user=self.user,
+            did=data
+        )
+
+        if not did.exists():
+            raise ValidationError("Did is not valid!")
+
+        self._did = did.first()
+            
+        return data
+
+    def clean_schema(self):
+        data = self.cleaned_data["schema"]
+        schema = Schemas.objects.filter(
+            id=data
+        )
+        if not schema.exists():
+            raise ValidationError("Schema is not valid!")
+
+        self._schema = schema.first()
+
+        return data
+
+    def clean_file_import(self):
+        data = self.cleaned_data["file_import"]
+        self.file_name = data.name
+        if File_datas.objects.filter(file_name=self.file_name, success=True).exists():
+            raise ValidationError("This file already exists!")
+
+        self.json_schema = json.loads(self._schema.data)
+        df = pd.read_csv (data, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL)
+        data_pd = df.fillna('').to_dict()
+
+        if not data_pd:
+            self.exception("This file is empty!")
+
+        for n in range(df.last_valid_index()+1):
+            row = {}
+            for k in data_pd.keys():
+                row[k] = data_pd[k][n]
+
+            user = self.validate_jsonld(n, row)
+            self.rows[user] = row
+
+        return data
+
+    def save(self, commit=True):
+        table = []
+        for k, v in self.rows.items():
+            table.append(self.create_credential(k, v)) 
+
+        if commit:
+            for cred in table:
+              cred.save()
+            File_datas.objects.create(file_name=self.file_name)
+            return table
+        
+        return 
+
+    def validate_jsonld(self, line, row):
+        try:
+            validate(instance=row, schema=self.json_schema)
+        except Exception as e:
+            msg = "line {}: {}".format(line+1, e)
+            self.exception(msg)
+
+        user = User.objects.filter(email=row.get('email'))
+        if not user:
+            txt = _('The user not exist!')
+            msg = "line {}: {}".format(line+1, txt)
+            self.exception(msg)
+
+        return user.first()
+
+    def create_credential(self, user, row):
+        d = copy.copy(self.json_schema)
+        d['instance'] = row
+        return VerificableCredential(
+            verified=False,
+            user=user,
+            data=json.dumps(d)
+        )
+
+    def exception(self, msg):
+        File_datas.objects.create(file_name=self.file_name, success=False)
+        raise ValidationError(msg)
+
 
 class SchemaForm(forms.Form):
     file_template = forms.FileField()

idhub/admin/views.py

@@ -1,5 +1,4 @@
 import os
-import csv
 import json
 import copy
 import logging
@@ -11,7 +10,12 @@ from smtplib import SMTPException
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 from django.views.generic.base import TemplateView
-from django.views.generic.edit import UpdateView, CreateView, DeleteView
+from django.views.generic.edit import (
+    CreateView,
+    DeleteView,
+    FormView,
+    UpdateView,
+)
 from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse_lazy
 from django.http import HttpResponse
@@ -61,7 +65,7 @@ class SchemasMix(AdminView, TemplateView):
     section = "Templates"
 
 
-class ImportExport(AdminView, TemplateView):
+class ImportExport(AdminView):
     title = _("Massive Data Management")
     section = "ImportExport"
 
@@ -695,7 +699,7 @@ class SchemasImportAddView(SchemasMix):
         return data
 
 
-class ImportView(ImportExport):
+class ImportView(ImportExport, TemplateView):
     template_name = "idhub/admin/import.html"
     subtitle = _('Import')
     icon = ''
@@ -708,7 +712,7 @@ class ImportView(ImportExport):
         return context
 
 
-class ImportStep2View(ImportExport):
+class ImportStep2View(ImportExport, TemplateView):
     template_name = "idhub/admin/import_step2.html"
     subtitle = _('Import')
     icon = ''
@@ -721,93 +725,23 @@ class ImportStep2View(ImportExport):
         return context
 
 
-class ImportStep3View(ImportExport):
+class ImportAddView(ImportExport, FormView):
-    template_name = "idhub/admin/import_step3.html"
+    template_name = "idhub/admin/import_add.html"
     subtitle = _('Import')
     icon = ''
+    form_class = ImportForm
     success_url = reverse_lazy('idhub:admin_import')
 
-    def get_context_data(self, **kwargs):
+    def get_form_kwargs(self):
-        context = super().get_context_data(**kwargs)
+        kwargs = super().get_form_kwargs()
-        context.update({
+        kwargs['user'] = self.request.user
-            'form': ImportForm(),
+        return kwargs
-        })
-        return context
 
-    def post(self, request, *args, **kwargs):
+    def form_valid(self, form):
-        self.pk = kwargs['pk']
+        cred = form.save()
-        self.schema = get_object_or_404(Schemas, pk=self.pk)
+        if cred:
-        form = ImportForm(request.POST, request.FILES)
+            messages.success(self.request, _("The file import was successfully!"))
-        if form.is_valid():
-            result = self.handle_uploaded_file()
-            if not result:
-                messages.error(request, _("There are some errors in the file"))
-                return super().get(request, *args, **kwargs)
-            return redirect(self.success_url)
         else:
-            return super().get(request, *args, **kwargs)
+            messages.error(self.request, _("Error importing the file!"))
-
+        return super().form_valid(form)
-        return super().post(request, *args, **kwargs)
-
-    def handle_uploaded_file(self):
-        f = self.request.FILES.get('file_import')
-        if not f:
-            messages.error(self.request, _("There aren't file"))
-            return
-
-        file_name = f.name
-        if File_datas.objects.filter(file_name=file_name, success=True).exists():
-            messages.error(self.request, _("This file already exists!"))
-            return
-
-        self.json_schema = json.loads(self.schema.data)
-        df = pd.read_csv (f, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL)
-        data_pd = df.fillna('').to_dict()
-        rows = {}
-
-        if not data_pd:
-            File_datas.objects.create(file_name=file_name, success=False)
-            return
-
-        for n in range(df.last_valid_index()+1):
-            row = {}
-            for k in data_pd.keys():
-                row[k] = data_pd[k][n]
-
-            user = self.validate(n, row)
-            if not user:
-                File_datas.objects.create(file_name=file_name, success=False)
-                return
-
-            rows[user] = row
-
-        File_datas.objects.create(file_name=file_name)
-        for k, v in rows.items():
-            self.create_credential(k, v)
-
-        return True
-
-    def validate(self, line, row):
-        try:
-            validate(instance=row, schema=self.json_schema)
-        except Exception as e:
-            messages.error(self.request, "line {}: {}".format(line+1, e))
-            return
-
-        user = User.objects.filter(email=row.get('email'))
-        if not user:
-            txt = _('The user not exist!')
-            messages.error(self.request, "line {}: {}".format(line+1, txt))
-            return
-
-        return user.first()
-
-    def create_credential(self, user, row):
-        d = copy.copy(self.json_schema)
-        d['instance'] = row
-        return VerificableCredential.objects.create(
-            verified=False,
-            user=user,
-            data=json.dumps(d)
-        )
 

idhub/templates/idhub/admin/import.html

@@ -28,7 +28,7 @@
         </tbody>
       </table>
       <div class="form-actions-no-box">
-        <a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step2' %}">{% translate "Import Datas" %} <i class="bi bi-plus"></i></a>
+        <a class="btn btn-green-admin" href="{% url 'idhub:admin_import_add' %}">{% translate "Import Datas" %} <i class="bi bi-plus"></i></a>
       </div>
     </div>
   </div>

idhub/templates/idhub/admin/import_add.html

@@ -6,29 +6,27 @@
   <i class="{{ icon }}"></i>
   {{ subtitle }}
 </h3>
-<div class="row mt-5">
+{% load django_bootstrap5 %}
-  <div class="col">
+<form role="form" method="post" enctype="multipart/form-data">
-    <div class="table-responsive">
+{% csrf_token %}
-      <table class="table table-striped table-sm">
+{% if form.errors %}
-        <thead>
+<div class="alert alert-danger alert-icon alert-icon-border alert-dismissible" role="alert">
-          <tr>
+  <div class="icon"><span class="mdi mdi-close-circle-o"></span></div>
-            <th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Created at' %}</button></th>
+  <div class="message">
-            <th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Template file' %}</button></th>
+    <button class="close" type="button" data-dismiss="alert" aria-label="Close">
-            <th scope="col"></th>
+    <span class="mdi mdi-close" aria-hidden="true"></span>
-            <th scope="col"></th>
+    </button>
-          </tr>
+    {% for field, error in form.errors.items %}
-        </thead>
+      {{ error }}
-        <tbody>
+    {% endfor %}
-          {% for schema in schemas.all %}
-          <tr style="font-size:15px;">
-            <td>{{ schema.created_at }}</td>
-            <td>{{ schema.file_schema }}</td>
-            <td><a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step3' schema.id %}" title="{% trans 'Import Dates' %}">{% trans 'Import Dates' %}</a></td>
-          </tr>
-          {% endfor %}
-        </tbody>
-      </table>
-    </div>
   </div>
 </div>
+{% endif %}
+{% bootstrap_form form %}
+<div class="form-actions-no-box">
+  <a class="btn btn-grey" href="{% url 'idhub:admin_import' %}">{% translate "Cancel" %}</a>
+  <input class="btn btn-green-admin" type="submit" name="submit" value="{% translate 'Save' %}" />
+</div>
+
+</form>
 {% endblock %}

idhub/templates/idhub/admin/import_step2.html

@@ -1,34 +0,0 @@
-{% extends "idhub/base_admin.html" %}
-{% load i18n %}
-
-{% block content %}
-<h3>
-  <i class="{{ icon }}"></i>
-  {{ subtitle }}
-</h3>
-<div class="row mt-5">
-  <div class="col">
-    <div class="table-responsive">
-      <table class="table table-striped table-sm">
-        <thead>
-          <tr>
-            <th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Created at' %}</button></th>
-            <th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Template file' %}</button></th>
-            <th scope="col"></th>
-            <th scope="col"></th>
-          </tr>
-        </thead>
-        <tbody>
-          {% for schema in schemas.all %}
-          <tr style="font-size:15px;">
-            <td>{{ schema.created_at }}</td>
-            <td>{{ schema.file_schema }}</td>
-            <td><a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step3' schema.id %}" title="{% trans 'Import Dates' %}">{% trans 'Import Dates' %}</a></td>
-          </tr>
-          {% endfor %}
-        </tbody>
-      </table>
-    </div>
-  </div>
-</div>
-{% endblock %}

idhub/templates/idhub/admin/import_step3.html

@@ -1,32 +0,0 @@
-{% extends "idhub/base_admin.html" %}
-{% load i18n %}
-
-{% block content %}
-<h3>
-  <i class="{{ icon }}"></i>
-  {{ subtitle }}
-</h3>
-{% load django_bootstrap5 %}
-<form role="form" method="post" enctype="multipart/form-data">
-{% csrf_token %}
-{% if form.errors %}
-<div class="alert alert-danger alert-icon alert-icon-border alert-dismissible" role="alert">
-  <div class="icon"><span class="mdi mdi-close-circle-o"></span></div>
-  <div class="message">
-    <button class="close" type="button" data-dismiss="alert" aria-label="Close">
-    <span class="mdi mdi-close" aria-hidden="true"></span>
-    </button>
-    {% for field, error in form.errors.items %}
-      {{ error }}
-    {% endfor %}
-  </div>
-</div>
-{% endif %}
-{% bootstrap_form form %}
-<div class="form-actions-no-box">
-  <a class="btn btn-grey" href="{% url 'idhub:admin_import' %}">{% translate "Cancel" %}</a>
-  <input class="btn btn-green-admin" type="submit" name="submit" value="{% translate 'Save' %}" />
-</div>
-
-</form>
-{% endblock %}

idhub/urls.py

@@ -169,8 +169,6 @@ urlpatterns = [
         name='admin_schemas_import_add'),
     path('admin/import', views_admin.ImportView.as_view(),
         name='admin_import'),
-    path('admin/import/new', views_admin.ImportStep2View.as_view(),
+    path('admin/import/new', views_admin.ImportAddView.as_view(),
-        name='admin_import_step2'),
+        name='admin_import_add'),
-    path('admin/import/<int:pk>/', views_admin.ImportStep3View.as_view(),
-        name='admin_import_step3'),
 ]