refactor form import csv
This commit is contained in:
Cayo Puigdefabregas
parent
f1c9608ca9
commit
d154617ca3
|
|
@ -1,9 +1,128 @@
|
|||
import csv
|
||||
import json
|
||||
import copy
|
||||
import pandas as pd
|
||||
from jsonschema import validate
|
||||
|
||||
from django import forms
|
||||
from django.core.exceptions import ValidationError
|
||||
from idhub.models import (
|
||||
DID,
|
||||
File_datas,
|
||||
Schemas,
|
||||
VerificableCredential,
|
||||
)
|
||||
from idhub_auth.models import User
|
||||
|
||||
|
||||
class ImportForm(forms.Form):
|
||||
did = forms.ChoiceField(choices=[])
|
||||
schema = forms.ChoiceField(choices=[])
|
||||
file_import = forms.FileField()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._schema = None
|
||||
self._did = None
|
||||
self.rows = {}
|
||||
self.user = kwargs.pop('user', None)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields['did'].choices = [
|
||||
(x.did, x.label) for x in DID.objects.filter(user=self.user)
|
||||
]
|
||||
self.fields['schema'].choices = [
|
||||
(x.id, x.name()) for x in Schemas.objects.filter()
|
||||
]
|
||||
|
||||
def clean_did(self):
|
||||
data = self.cleaned_data["did"]
|
||||
did = DID.objects.filter(
|
||||
user=self.user,
|
||||
did=data
|
||||
)
|
||||
|
||||
if not did.exists():
|
||||
raise ValidationError("Did is not valid!")
|
||||
|
||||
self._did = did.first()
|
||||
|
||||
return data
|
||||
|
||||
def clean_schema(self):
|
||||
data = self.cleaned_data["schema"]
|
||||
schema = Schemas.objects.filter(
|
||||
id=data
|
||||
)
|
||||
if not schema.exists():
|
||||
raise ValidationError("Schema is not valid!")
|
||||
|
||||
self._schema = schema.first()
|
||||
|
||||
return data
|
||||
|
||||
def clean_file_import(self):
|
||||
data = self.cleaned_data["file_import"]
|
||||
self.file_name = data.name
|
||||
if File_datas.objects.filter(file_name=self.file_name, success=True).exists():
|
||||
raise ValidationError("This file already exists!")
|
||||
|
||||
self.json_schema = json.loads(self._schema.data)
|
||||
df = pd.read_csv (data, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
data_pd = df.fillna('').to_dict()
|
||||
|
||||
if not data_pd:
|
||||
self.exception("This file is empty!")
|
||||
|
||||
for n in range(df.last_valid_index()+1):
|
||||
row = {}
|
||||
for k in data_pd.keys():
|
||||
row[k] = data_pd[k][n]
|
||||
|
||||
user = self.validate_jsonld(n, row)
|
||||
self.rows[user] = row
|
||||
|
||||
return data
|
||||
|
||||
def save(self, commit=True):
|
||||
table = []
|
||||
for k, v in self.rows.items():
|
||||
table.append(self.create_credential(k, v))
|
||||
|
||||
if commit:
|
||||
for cred in table:
|
||||
cred.save()
|
||||
File_datas.objects.create(file_name=self.file_name)
|
||||
return table
|
||||
|
||||
return
|
||||
|
||||
def validate_jsonld(self, line, row):
|
||||
try:
|
||||
validate(instance=row, schema=self.json_schema)
|
||||
except Exception as e:
|
||||
msg = "line {}: {}".format(line+1, e)
|
||||
self.exception(msg)
|
||||
|
||||
user = User.objects.filter(email=row.get('email'))
|
||||
if not user:
|
||||
txt = _('The user not exist!')
|
||||
msg = "line {}: {}".format(line+1, txt)
|
||||
self.exception(msg)
|
||||
|
||||
return user.first()
|
||||
|
||||
def create_credential(self, user, row):
|
||||
d = copy.copy(self.json_schema)
|
||||
d['instance'] = row
|
||||
return VerificableCredential(
|
||||
verified=False,
|
||||
user=user,
|
||||
data=json.dumps(d)
|
||||
)
|
||||
|
||||
def exception(self, msg):
|
||||
File_datas.objects.create(file_name=self.file_name, success=False)
|
||||
raise ValidationError(msg)
|
||||
|
||||
|
||||
class SchemaForm(forms.Form):
|
||||
file_template = forms.FileField()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import csv
|
||||
import json
|
||||
import copy
|
||||
import logging
|
||||
|
|
@ -11,7 +10,12 @@ from smtplib import SMTPException
|
|||
from django.conf import settings
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django.views.generic.base import TemplateView
|
||||
from django.views.generic.edit import UpdateView, CreateView, DeleteView
|
||||
from django.views.generic.edit import (
|
||||
CreateView,
|
||||
DeleteView,
|
||||
FormView,
|
||||
UpdateView,
|
||||
)
|
||||
from django.shortcuts import get_object_or_404, redirect
|
||||
from django.urls import reverse_lazy
|
||||
from django.http import HttpResponse
|
||||
|
|
@ -61,7 +65,7 @@ class SchemasMix(AdminView, TemplateView):
|
|||
section = "Templates"
|
||||
|
||||
|
||||
class ImportExport(AdminView, TemplateView):
|
||||
class ImportExport(AdminView):
|
||||
title = _("Massive Data Management")
|
||||
section = "ImportExport"
|
||||
|
||||
|
|
@ -695,7 +699,7 @@ class SchemasImportAddView(SchemasMix):
|
|||
return data
|
||||
|
||||
|
||||
class ImportView(ImportExport):
|
||||
class ImportView(ImportExport, TemplateView):
|
||||
template_name = "idhub/admin/import.html"
|
||||
subtitle = _('Import')
|
||||
icon = ''
|
||||
|
|
@ -708,7 +712,7 @@ class ImportView(ImportExport):
|
|||
return context
|
||||
|
||||
|
||||
class ImportStep2View(ImportExport):
|
||||
class ImportStep2View(ImportExport, TemplateView):
|
||||
template_name = "idhub/admin/import_step2.html"
|
||||
subtitle = _('Import')
|
||||
icon = ''
|
||||
|
|
@ -721,93 +725,23 @@ class ImportStep2View(ImportExport):
|
|||
return context
|
||||
|
||||
|
||||
class ImportStep3View(ImportExport):
|
||||
template_name = "idhub/admin/import_step3.html"
|
||||
class ImportAddView(ImportExport, FormView):
|
||||
template_name = "idhub/admin/import_add.html"
|
||||
subtitle = _('Import')
|
||||
icon = ''
|
||||
form_class = ImportForm
|
||||
success_url = reverse_lazy('idhub:admin_import')
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
context.update({
|
||||
'form': ImportForm(),
|
||||
})
|
||||
return context
|
||||
def get_form_kwargs(self):
|
||||
kwargs = super().get_form_kwargs()
|
||||
kwargs['user'] = self.request.user
|
||||
return kwargs
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
self.pk = kwargs['pk']
|
||||
self.schema = get_object_or_404(Schemas, pk=self.pk)
|
||||
form = ImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
result = self.handle_uploaded_file()
|
||||
if not result:
|
||||
messages.error(request, _("There are some errors in the file"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
return redirect(self.success_url)
|
||||
def form_valid(self, form):
|
||||
cred = form.save()
|
||||
if cred:
|
||||
messages.success(self.request, _("The file import was successfully!"))
|
||||
else:
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
return super().post(request, *args, **kwargs)
|
||||
|
||||
def handle_uploaded_file(self):
|
||||
f = self.request.FILES.get('file_import')
|
||||
if not f:
|
||||
messages.error(self.request, _("There aren't file"))
|
||||
return
|
||||
|
||||
file_name = f.name
|
||||
if File_datas.objects.filter(file_name=file_name, success=True).exists():
|
||||
messages.error(self.request, _("This file already exists!"))
|
||||
return
|
||||
|
||||
self.json_schema = json.loads(self.schema.data)
|
||||
df = pd.read_csv (f, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
data_pd = df.fillna('').to_dict()
|
||||
rows = {}
|
||||
|
||||
if not data_pd:
|
||||
File_datas.objects.create(file_name=file_name, success=False)
|
||||
return
|
||||
|
||||
for n in range(df.last_valid_index()+1):
|
||||
row = {}
|
||||
for k in data_pd.keys():
|
||||
row[k] = data_pd[k][n]
|
||||
|
||||
user = self.validate(n, row)
|
||||
if not user:
|
||||
File_datas.objects.create(file_name=file_name, success=False)
|
||||
return
|
||||
|
||||
rows[user] = row
|
||||
|
||||
File_datas.objects.create(file_name=file_name)
|
||||
for k, v in rows.items():
|
||||
self.create_credential(k, v)
|
||||
|
||||
return True
|
||||
|
||||
def validate(self, line, row):
|
||||
try:
|
||||
validate(instance=row, schema=self.json_schema)
|
||||
except Exception as e:
|
||||
messages.error(self.request, "line {}: {}".format(line+1, e))
|
||||
return
|
||||
|
||||
user = User.objects.filter(email=row.get('email'))
|
||||
if not user:
|
||||
txt = _('The user not exist!')
|
||||
messages.error(self.request, "line {}: {}".format(line+1, txt))
|
||||
return
|
||||
|
||||
return user.first()
|
||||
|
||||
def create_credential(self, user, row):
|
||||
d = copy.copy(self.json_schema)
|
||||
d['instance'] = row
|
||||
return VerificableCredential.objects.create(
|
||||
verified=False,
|
||||
user=user,
|
||||
data=json.dumps(d)
|
||||
)
|
||||
messages.error(self.request, _("Error importing the file!"))
|
||||
return super().form_valid(form)
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
</tbody>
|
||||
</table>
|
||||
<div class="form-actions-no-box">
|
||||
<a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step2' %}">{% translate "Import Datas" %} <i class="bi bi-plus"></i></a>
|
||||
<a class="btn btn-green-admin" href="{% url 'idhub:admin_import_add' %}">{% translate "Import Datas" %} <i class="bi bi-plus"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -6,29 +6,27 @@
|
|||
<i class="{{ icon }}"></i>
|
||||
{{ subtitle }}
|
||||
</h3>
|
||||
<div class="row mt-5">
|
||||
<div class="col">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Created at' %}</button></th>
|
||||
<th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Template file' %}</button></th>
|
||||
<th scope="col"></th>
|
||||
<th scope="col"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for schema in schemas.all %}
|
||||
<tr style="font-size:15px;">
|
||||
<td>{{ schema.created_at }}</td>
|
||||
<td>{{ schema.file_schema }}</td>
|
||||
<td><a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step3' schema.id %}" title="{% trans 'Import Dates' %}">{% trans 'Import Dates' %}</a></td>
|
||||
</tr>
|
||||
{% load django_bootstrap5 %}
|
||||
<form role="form" method="post" enctype="multipart/form-data">
|
||||
{% csrf_token %}
|
||||
{% if form.errors %}
|
||||
<div class="alert alert-danger alert-icon alert-icon-border alert-dismissible" role="alert">
|
||||
<div class="icon"><span class="mdi mdi-close-circle-o"></span></div>
|
||||
<div class="message">
|
||||
<button class="close" type="button" data-dismiss="alert" aria-label="Close">
|
||||
<span class="mdi mdi-close" aria-hidden="true"></span>
|
||||
</button>
|
||||
{% for field, error in form.errors.items %}
|
||||
{{ error }}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% bootstrap_form form %}
|
||||
<div class="form-actions-no-box">
|
||||
<a class="btn btn-grey" href="{% url 'idhub:admin_import' %}">{% translate "Cancel" %}</a>
|
||||
<input class="btn btn-green-admin" type="submit" name="submit" value="{% translate 'Save' %}" />
|
||||
</div>
|
||||
|
||||
</form>
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
{% extends "idhub/base_admin.html" %}
|
||||
{% load i18n %}
|
||||
|
||||
{% block content %}
|
||||
<h3>
|
||||
<i class="{{ icon }}"></i>
|
||||
{{ subtitle }}
|
||||
</h3>
|
||||
<div class="row mt-5">
|
||||
<div class="col">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Created at' %}</button></th>
|
||||
<th scope="col"><button type="button" class="btn btn-grey border border-dark">{% trans 'Template file' %}</button></th>
|
||||
<th scope="col"></th>
|
||||
<th scope="col"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for schema in schemas.all %}
|
||||
<tr style="font-size:15px;">
|
||||
<td>{{ schema.created_at }}</td>
|
||||
<td>{{ schema.file_schema }}</td>
|
||||
<td><a class="btn btn-green-admin" href="{% url 'idhub:admin_import_step3' schema.id %}" title="{% trans 'Import Dates' %}">{% trans 'Import Dates' %}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
{% extends "idhub/base_admin.html" %}
|
||||
{% load i18n %}
|
||||
|
||||
{% block content %}
|
||||
<h3>
|
||||
<i class="{{ icon }}"></i>
|
||||
{{ subtitle }}
|
||||
</h3>
|
||||
{% load django_bootstrap5 %}
|
||||
<form role="form" method="post" enctype="multipart/form-data">
|
||||
{% csrf_token %}
|
||||
{% if form.errors %}
|
||||
<div class="alert alert-danger alert-icon alert-icon-border alert-dismissible" role="alert">
|
||||
<div class="icon"><span class="mdi mdi-close-circle-o"></span></div>
|
||||
<div class="message">
|
||||
<button class="close" type="button" data-dismiss="alert" aria-label="Close">
|
||||
<span class="mdi mdi-close" aria-hidden="true"></span>
|
||||
</button>
|
||||
{% for field, error in form.errors.items %}
|
||||
{{ error }}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% bootstrap_form form %}
|
||||
<div class="form-actions-no-box">
|
||||
<a class="btn btn-grey" href="{% url 'idhub:admin_import' %}">{% translate "Cancel" %}</a>
|
||||
<input class="btn btn-green-admin" type="submit" name="submit" value="{% translate 'Save' %}" />
|
||||
</div>
|
||||
|
||||
</form>
|
||||
{% endblock %}
|
||||
|
|
@ -169,8 +169,6 @@ urlpatterns = [
|
|||
name='admin_schemas_import_add'),
|
||||
path('admin/import', views_admin.ImportView.as_view(),
|
||||
name='admin_import'),
|
||||
path('admin/import/new', views_admin.ImportStep2View.as_view(),
|
||||
name='admin_import_step2'),
|
||||
path('admin/import/<int:pk>/', views_admin.ImportStep3View.as_view(),
|
||||
name='admin_import_step3'),
|
||||
path('admin/import/new', views_admin.ImportAddView.as_view(),
|
||||
name='admin_import_add'),
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in New Issue