pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Optimized save model operations with update_fields kwarg

Browse source
This commit is contained in:
Marc 2014-09-30 16:06:42 +00:00
parent 5ed8773acc
commit 774422a41b
19 changed files with 214 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
TODO.md
View file

@ -138,3 +138,5 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* Redirect junk emails and delete every 30 days?
* Complitely decouples scripts execution, billing, service definition

121
orchestra/apps/accounts/backends.py Normal file
View file

@ -0,0 +1,121 @@
import textwrap
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
# TODO create a base backend for SystemUsers!
class MainUserBackend(ServiceController):
verbose_name = _("Main user")
model = 'accounts.Account'
ignore_fields = ['last_login']
def save(self, user):
context = self.get_context(user)
self.append(textwrap.dedent("""
if [[ $( id %(username)s ) ]]; then
usermod --password '%(password)s' %(username)s
else
useradd %(username)s --password '%(password)s' --shell %(shell)s
fi
mkdir -p %(home)s
chown %(username)s.%(username)s %(home)s""" % context
))
def delete(self, user):
context = self.get_context(user)
self.append("{ sleep 2 && killall -u %(username)s -s KILL; } &" % context)
self.append("killall -u %(username)s" % context)
self.append("userdel %(username)s" % context)
def get_context(self, user):
context = {
'username': user.username,
'password': user.password if user.is_active else '*%s' % user.password,
'shell': getattr(user, 'shell', settings.ACCOUNTS_DEFAULT_SHELL)
}
context['home'] = settings.ACCOUNTS_HOME % context
return context
class MainUserDisk(ServiceMonitor):
model = 'accounts.Account'
resource = ServiceMonitor.DISK
verbose_name = _('Main user disk')
def monitor(self, user):
context = self.get_context(user)
self.append("du -s %(home)s | xargs echo %(object_id)s" % context)
def get_context(self, user):
context = SystemUserBackend().get_context(user)
context['object_id'] = user.pk
return context
class MainFTPTraffic(ServiceMonitor):
model = 'accounts.Account'
resource = ServiceMonitor.TRAFFIC
verbose_name = _('Main FTP traffic')
def prepare(self):
current_date = timezone.localtime(self.current_date)
current_date = current_date.strftime("%Y%m%d%H%M%S")
self.append(textwrap.dedent("""
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
USERNAME="$3"
LOG_FILE="$4"
grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \\
| grep " \\[${USERNAME}\\] " \\
| awk -v ini="${INI_DATE}" '
BEGIN {
end = "%s"
sum = 0
months["Jan"] = "01"
months["Feb"] = "02"
months["Mar"] = "03"
months["Apr"] = "04"
months["May"] = "05"
months["Jun"] = "06"
months["Jul"] = "07"
months["Aug"] = "08"
months["Sep"] = "09"
months["Oct"] = "10"
months["Nov"] = "11"
months["Dec"] = "12"
} {
# log: Fri Jul 11 13:23:17 2014
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
} END {
print sum
}
' | xargs echo ${OBJECT_ID}
}""" % current_date))
def monitor(self, user):
context = self.get_context(user)
self.append(
'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context)
def get_context(self, user):
last_date = timezone.localtime(self.get_last_date(user.pk))
return {
'log_file': settings.ACCOUNTS_FTP_LOG_PATH,
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
'object_id': user.pk,
'username': user.username,
}

5
orchestra/apps/accounts/forms.py
View file

@ -24,7 +24,7 @@ class AccountCreationForm(auth.forms.UserCreationForm):
class AccountChangeForm(forms.ModelForm):
username = forms.CharField()
username = forms.CharField(required=False)
password = auth.forms.ReadOnlyPasswordHashField(label=_("Password"),
help_text=_("Raw passwords are not stored, so there is no way to see "
"this user's password, but you can change the password "
@ -33,7 +33,8 @@ class AccountChangeForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
super(AccountChangeForm, self).__init__(*args, **kwargs)
account = kwargs.get('instance')
self.fields['username'].widget = ReadOnlyWidget(account.username)
username = '<b style="font-size:small">%s</b>' % account.username
self.fields['username'].widget = ReadOnlyWidget(username)
self.fields['password'].initial = account.password
def clean_password(self):

3
orchestra/apps/accounts/models.py
View file

@ -68,8 +68,7 @@ class Account(auth.AbstractBaseUser):
def send_email(self, template, context, contacts=[], attachments=[], html=None):
contacts = self.contacts.filter(email_usages=contacts)
email_to = contacts.values_list('email', flat=True)
send_email_template(template, context, email_to, html=html,
attachments=attachments)
send_email_template(template, context, email_to, html=html, attachments=attachments)
def get_full_name(self):
full_name = '%s %s' % (self.first_name, self.last_name)

9
orchestra/apps/accounts/settings.py
View file

@ -22,3 +22,12 @@ ACCOUNTS_DEFAULT_LANGUAGE = getattr(settings, 'ACCOUNTS_DEFAULT_LANGUAGE', 'en')
ACCOUNTS_MAIN_PK = getattr(settings, 'ACCOUNTS_MAIN_PK', 1)
ACCOUNTS_HOME = getattr(settings, 'ACCOUNTS_HOME', '/home/%(username)s')
ACCOUNTS_FTP_LOG_PATH = getattr(settings, 'ACCOUNTS_FTP_LOG_PATH', '/var/log/vsftpd.log')
ACCOUNTS_DEFAULT_SHELL = getattr(settings, 'ACCOUNTS_DEFAULT_SHELL', '/bin/false')

6
orchestra/apps/bills/models.py
View file

@ -156,7 +156,7 @@ class Bill(models.Model):
]
)
self.is_sent = True
self.save()
self.save(update_fields=['is_sent'])
def render(self, payment=False):
if payment is False:
@ -281,7 +281,7 @@ class BillLine(models.Model):
super(BillLine, self).save(*args, **kwargs)
if self.bill.is_open:
self.bill.total = self.bill.get_total()
self.bill.save()
self.bill.save(update_fields=['total'])
class BillSubline(models.Model):
@ -306,7 +306,7 @@ class BillSubline(models.Model):
super(BillSubline, self).save(*args, **kwargs)
if self.line.bill.is_open:
self.line.bill.total = self.line.bill.get_total()
self.line.bill.save()
self.line.bill.save(update_fields=['total'])
accounts.register(Bill)

4
orchestra/apps/domains/models.py
View file

@ -58,7 +58,7 @@ class Domain(models.Model):
serial = str(self.serial)[:8] + '%.2d' % num
serial = int(serial)
self.serial = serial
self.save()
self.save(update_fields=['serial'])
def render_records(self):
types = {}
@ -128,7 +128,7 @@ class Domain(models.Model):
domains = Domain.objects.exclude(pk=self.pk)
for domain in domains.filter(name__endswith=self.name):
domain.top = self
domain.save()
domain.save(update_fields=['top'])
self.get_subdomains().update(account=self.account)
def get_top(self):

2
orchestra/apps/issues/actions.py
View file

@ -122,6 +122,6 @@ def set_default_queue(modeladmin, request, queryset):
Queue.objects.filter(default=True).update(default=False)
queue = queryset.get()
queue.default = True
queue.save()
queue.save(update_fields=['default'])
modeladmin.log_change(request, queue, _("Chosen as default."))
messages.info(request, _("Chosen '%s' as default queue.") % queue)

8
orchestra/apps/issues/models.py
View file

@ -135,19 +135,19 @@ class Ticket(models.Model):
def reject(self):
self.state = Ticket.REJECTED
self.save()
self.save(update_fields=['state'])
def resolve(self):
self.state = Ticket.RESOLVED
self.save()
self.save(update_fields=['state'])
def close(self):
self.state = Ticket.CLOSED
self.save()
self.save(update_fields=['state'])
def take(self, user):
self.owner = user
self.save()
self.save(update_fields=['state'])
class Message(models.Model):

74
orchestra/apps/mails/backends.py
View file

@ -1,3 +1,4 @@
import textwrap
import os
from django.utils import timezone
@ -18,14 +19,13 @@ class MailSystemUserBackend(ServiceController):
DEFAULT_GROUP = 'postfix'
def create_user(self, context):
self.append(
"if [[ $( id %(username)s ) ]]; then \n"
" usermod -p '%(password)s' %(username)s \n"
"else \n"
" useradd %(username)s --password '%(password)s' \\\n"
" --shell /dev/null \n"
"fi" % context
)
self.append(textwrap("""
if [[ $( id %(username)s ) ]]; then
usermod -p '%(password)s' %(username)s
else
useradd %(username)s --password '%(password)s' --shell /dev/null
fi""" % context
))
self.append("mkdir -p %(home)s" % context)
self.append("chown %(username)s.%(group)s %(home)s" % context)
@ -51,12 +51,12 @@ class MailSystemUserBackend(ServiceController):
'quota': mailbox.resources.disk.allocated*1000*1000,
})
self.append("mkdir -p %(maildir_path)s" % context)
self.append(
"sed -i '1s/.*/%(quota)s,S/' %(maildirsize_path)s || {"
" echo '%(quota)s,S' > %(maildirsize_path)s && "
" chown %(username)s %(maildirsize_path)s;"
"}" % context
)
self.append(textwrap("""
sed -i '1s/.*/%(quota)s,S/' %(maildirsize_path)s || {
echo '%(quota)s,S' > %(maildirsize_path)s &&
chown %(username)s %(maildirsize_path)s;
}""" % context
))
def save(self, mailbox):
context = self.get_context(mailbox)
@ -98,26 +98,26 @@ class PostfixAddressBackend(ServiceController):
self.append('sed -i "s/^%(domain)s//" %(virtdomains)s' % context)
def update_virtusertable(self, context):
self.append(
'LINE="%(email)s\t%(destination)s"\n'
'if [[ ! $(grep "^%(email)s\s" %(virtusertable)s) ]]; then\n'
' echo "$LINE" >> %(virtusertable)s\n'
' UPDATED=1\n'
'else\n'
' if [[ ! $(grep "^${LINE}$" %(virtusertable)s) ]]; then\n'
' sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtusertable)s\n'
' UPDATED=1\n'
' fi\n'
'fi' % context
)
self.append(textwrap("""
LINE="%(email)s\t%(destination)s"
if [[ ! $(grep "^%(email)s\s" %(virtusertable)s) ]]; then
echo "$LINE" >> %(virtusertable)s
UPDATED=1
else
if [[ ! $(grep "^${LINE}$" %(virtusertable)s) ]]; then
sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtusertable)s
UPDATED=1
fi
fi""" % context
))
def exclude_virtusertable(self, context):
self.append(
'if [[ $(grep "^%(email)s\s") ]]; then\n'
' sed -i "s/^%(email)s\s.*$//" %(virtusertable)s\n'
' UPDATED=1\n'
'fi'
)
self.append(textwrap("""
if [[ $(grep "^%(email)s\s") ]]; then
sed -i "s/^%(email)s\s.*$//" %(virtusertable)s
UPDATED=1
fi"""
))
def save(self, address):
context = self.get_context(address)
@ -131,10 +131,12 @@ class PostfixAddressBackend(ServiceController):
def commit(self):
context = self.get_context_files()
self.append('[[ $UPDATED == 1 ]] && { '
'postmap %(virtdomains)s;'
'postmap %(virtusertable)s;'
'}' % context)
self.append(textwrap("""
[[ $UPDATED == 1 ]] && {
postmap %(virtdomains)s
postmap %(virtusertable)s
}""" % context
))
def get_context_files(self):
return {

8
orchestra/apps/orchestration/methods.py
View file

@ -16,7 +16,7 @@ def BashSSH(backend, log, server, cmds):
script = '\n'.join(['set -e', 'set -o pipefail'] + cmds + ['exit 0'])
script = script.replace('\r', '')
log.script = script
log.save()
log.save(update_fields=['script'])
try:
# Avoid "Argument list too long" on large scripts by genereting a file
@ -34,7 +34,7 @@ def BashSSH(backend, log, server, cmds):
key_filename=settings.ORCHESTRATION_SSH_KEY_PATH)
except socket.error:
log.state = BackendLog.TIMEOUT
log.save()
log.save(update_fields=['state'])
return
transport = ssh.get_transport()
channel = transport.open_session()
@ -66,7 +66,7 @@ def BashSSH(backend, log, server, cmds):
log.stdout += channel.recv(1024)
if channel.recv_stderr_ready():
log.stderr += channel.recv_stderr(1024)
log.save()
log.save(update_fields=['stdout', 'stderr'])
if channel.exit_status_ready():
break
log.exit_code = exit_code = channel.recv_exit_status()
@ -85,7 +85,7 @@ def Python(backend, log, server, cmds):
script = [ str(cmd.func.func_name) + str(cmd.args) for cmd in cmds ]
script = json.dumps(script, indent=4).replace('"', '')
log.script = '\n'.join([log.script, script])
log.save()
log.save(update_fields=['script'])
stdout = ''
try:
for cmd in cmds:

6
orchestra/apps/orders/models.py
View file

@ -162,11 +162,11 @@ class Order(models.Model):
id=self.id, description=description, metric=metric))
if self.description != description:
self.description = description
self.save()
self.save(update_fields=['description'])
def cancel(self):
self.cancelled_on = timezone.now()
self.save()
self.save(update_fields=['cancelled_on'])
logger.info("CANCELLED order id: {id}".format(id=self.id))
def get_metric(self, *args, **kwargs):
@ -230,7 +230,7 @@ class MetricStorage(models.Model):
cls.objects.create(order=order, value=value, updated_on=now)
else:
metric.updated_on = now
metric.save()
metric.save(update_fields=['updated_on'])
accounts.register(Order)

6
orchestra/apps/payments/methods/sepadirectdebit.py
View file

@ -178,7 +178,7 @@ class SEPADirectDebit(PaymentMethod):
account = transaction.account
data = transaction.source.data
transaction.state = transaction.WAITTING_CONFIRMATION
transaction.save()
transaction.save(update_fields=['state'])
yield E.DrctDbtTxInf( # Direct Debit Transaction Info
E.PmtId( # Payment Id
E.EndToEndId(str(transaction.id)) # Payment Id/End to End
@ -219,7 +219,7 @@ class SEPADirectDebit(PaymentMethod):
account = transaction.account
data = transaction.source.data
transaction.state = transaction.WAITTING_CONFIRMATION
transaction.save()
transaction.save(update_fields=['state'])
yield E.CdtTrfTxInf( # Credit Transfer Transaction Info
E.PmtId( # Payment Id
E.EndToEndId(str(transaction.id)) # Payment Id/End to End
@ -278,7 +278,7 @@ class SEPADirectDebit(PaymentMethod):
sepa = etree.parse(StringIO(etree.tostring(sepa)))
schema.assertValid(sepa)
process.file = file_name
process.save()
process.save(update_fields=['file'])
sepa.write(process.file.path,
pretty_print=True,
xml_declaration=True,

14
orchestra/apps/payments/models.py
View file

@ -119,22 +119,22 @@ class Transaction(models.Model):
def mark_as_processed(self):
assert self.state == self.WAITTING_PROCESSING
self.state = self.WAITTING_EXECUTION
self.save()
self.save(update_fields=['state'])
def mark_as_executed(self):
assert self.state == self.WAITTING_EXECUTION
self.state = self.EXECUTED
self.save()
self.save(update_fields=['state'])
def mark_as_secured(self):
assert self.state == self.EXECUTED
self.state = self.SECURED
self.save()
self.save(update_fields=['state'])
def mark_as_rejected(self):
assert self.state == self.EXECUTED
self.state = self.REJECTED
self.save()
self.save(update_fields=['state'])
class TransactionProcess(models.Model):
@ -169,21 +169,21 @@ class TransactionProcess(models.Model):
self.state = self.EXECUTED
for transaction in self.transactions.all():
transaction.mark_as_executed()
self.save()
self.save(update_fields=['state'])
def abort(self):
assert self.state in [self.CREATED, self.EXCECUTED]
self.state = self.ABORTED
for transaction in self.transaction.all():
transaction.mark_as_aborted()
self.save()
self.save(update_fields=['state'])
def commit(self):
assert self.state in [self.CREATED, self.EXECUTED]
self.state = self.COMMITED
for transaction in self.transactions.processing():
transaction.mark_as_secured()
self.save()
self.save(update_fields=['state'])
accounts.register(PaymentSource)

4
orchestra/apps/resources/models.py
View file

@ -98,7 +98,7 @@ class Resource(models.Model):
task.delete()
elif task.crontab != self.crontab:
task.crontab = self.crontab
task.save()
task.save(update_fields=['crontab'])
if created:
# This only work on tests because of multiprocessing used on real deployments
create_resource_relation()
@ -146,7 +146,7 @@ class ResourceData(models.Model):
current = self.get_used()
self.used = current or 0
self.updated_at = timezone.now()
self.save()
self.save(update_fields=['used', 'updated_at'])
class MonitorData(models.Model):

4
orchestra/apps/services/handlers.py
View file

@ -222,7 +222,7 @@ class ServiceHandler(plugins.Plugin):
for order in givers:
if hasattr(order, 'new_billed_until'):
order.billed_until = order.new_billed_until
order.save()
order.save(update_fields=['billed_until'])
def apply_compensations(self, order, only_beyond=False):
dsize = 0
@ -481,5 +481,5 @@ class ServiceHandler(plugins.Plugin):
order = line.order
order.billed_on = now
order.billed_until = getattr(order, 'new_billed_until', order.billed_until)
order.save()
order.save(update_fields=['billed_on', 'billed_until'])
return lines

122
orchestra/apps/systemusers/backends.py
View file

@ -3,121 +3,15 @@ import textwrap
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
from orchestra.apps.accounts.backends import MainUserBackend, MainFTPTraffic
class SystemUserBackend(ServiceController):
verbose_name = _("System User")
model = 'users.User'
ignore_fields = ['last_login']
def save(self, user):
context = self.get_context(user)
if user.is_main:
self.append(textwrap.dedent("""
if [[ $( id %(username)s ) ]]; then
usermod --password '%(password)s' %(username)s
else
useradd %(username)s --password '%(password)s' \\
--shell /bin/false
fi
mkdir -p %(home)s
chown %(username)s.%(username)s %(home)s""" % context
))
else:
self.delete(user)
def delete(self, user):
context = self.get_context(user)
self.append("{ sleep 2 && killall -u %(username)s -s KILL; } &" % context)
self.append("killall -u %(username)s" % context)
self.append("userdel %(username)s" % context)
def get_context(self, user):
context = {
'username': user.username,
'password': user.password if user.is_active else '*%s' % user.password,
}
context['home'] = settings.USERS_SYSTEMUSER_HOME % context
return context
class SystemUserBackend(MainUserBackend):
verbose_name = _("System user")
model = 'systemusers.SystemUser'
ignore_fields = []
class SystemUserDisk(ServiceMonitor):
model = 'users.User'
resource = ServiceMonitor.DISK
verbose_name = _('System user disk')
def monitor(self, user):
context = self.get_context(user)
self.append("du -s %(home)s | xargs echo %(object_id)s" % context)
def get_context(self, user):
context = SystemUserBackend().get_context(user)
context['object_id'] = user.pk
return context
class FTPTraffic(ServiceMonitor):
model = 'users.User'
resource = ServiceMonitor.TRAFFIC
verbose_name = _('FTP traffic')
def prepare(self):
current_date = timezone.localtime(self.current_date)
current_date = current_date.strftime("%Y%m%d%H%M%S")
self.append(textwrap.dedent("""
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
USERNAME="$3"
LOG_FILE="$4"
grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \\
| grep " \\[${USERNAME}\\] " \\
| awk -v ini="${INI_DATE}" '
BEGIN {
end = "%s"
sum = 0
months["Jan"] = "01"
months["Feb"] = "02"
months["Mar"] = "03"
months["Apr"] = "04"
months["May"] = "05"
months["Jun"] = "06"
months["Jul"] = "07"
months["Aug"] = "08"
months["Sep"] = "09"
months["Oct"] = "10"
months["Nov"] = "11"
months["Dec"] = "12"
} {
# log: Fri Jul 11 13:23:17 2014
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
} END {
print sum
}
' | xargs echo ${OBJECT_ID}
}""" % current_date))
def monitor(self, user):
context = self.get_context(user)
self.append(
'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context)
def get_context(self, user):
last_date = timezone.localtime(self.get_last_date(user.pk))
return {
'log_file': settings.USERS_FTP_LOG_PATH,
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
'object_id': user.pk,
'username': user.username,
}
class SystemUserFTPTraffic(MainFTPTraffic):
model = 'systemusers.SystemUser'
verbose_name = _('System user FTP traffic')

11
orchestra/apps/systemusers/models.py
View file

@ -13,7 +13,7 @@ class SystemUserQuerySet(models.QuerySet):
def create_user(self, username, password='', **kwargs):
user = super(SystemUserQuerySet, self).create(username=username, **kwargs)
user.set_password(password)
user.save()
user.save(update_fields=['password'])
class SystemUser(models.Model):
@ -59,15 +59,6 @@ class SystemUser(models.Model):
def set_password(self, raw_password):
self.password = make_password(raw_password)
def check_password(self, raw_password):
"""
Returns a boolean of whether the raw_password was correct. Handles
hashing formats behind the scenes.
"""
def setter(raw_password):
self.set_password(raw_password)
self.save(update_fields=["password"])
def get_is_active(self):
return self.account.is_active and self.is_active

4
orchestra/apps/systemusers/settings.py
View file

@ -3,10 +3,6 @@ from django.conf import settings
from django.utils.translation import ugettext, ugettext_lazy as _
USERS_SYSTEMUSER_HOME = getattr(settings, 'USERES_SYSTEMUSER_HOME', '/home/%(username)s')
USERS_FTP_LOG_PATH = getattr(settings, 'USERS_FTP_LOG_PATH', '/var/log/vsftpd.log')
USERS_SHELLS = getattr(settings, 'USERS_SHELLS', (
('/bin/false', _("No shell, FTP only")),
('/bin/rsync', _("No shell, SFTP/RSYNC only")),