Fixes on resources, mailer and vps0
This commit is contained in:
Marc Aymerich
parent
99071f01b1
commit
4593fcc278
48
TODO.md
48
TODO.md
|
|
@ -36,8 +36,6 @@
|
|||
|
||||
* What fields we really need on contacts? name email phone and what more?
|
||||
|
||||
* Redirect junk emails and delete every 30 days?
|
||||
|
||||
* DOC: Complitely decouples scripts execution, billing, service definition
|
||||
|
||||
* init.d celery scripts
|
||||
|
|
@ -58,21 +56,11 @@
|
|||
* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture --keepdb
|
||||
|
||||
* ForeignKey.swappable
|
||||
* Field.editable
|
||||
* ManyToManyField.symmetrical = False (user group)
|
||||
|
||||
* REST PERMISSIONS
|
||||
|
||||
* caching based on "def text2int(textnum, numwords={}):"
|
||||
|
||||
* sync() ServiceController method that synchronizes orchestra and servers (delete or import)
|
||||
|
||||
* consider removing mailbox support on forward (user@pangea.org instead)
|
||||
|
||||
* Databases.User add reverse M2M databases widget (like mailbox.addresses)
|
||||
|
||||
* Grant permissions to systemusers
|
||||
|
||||
* Make one dedicated CGI user for each account only for CGI execution (fpm/fcgid). Different from the files owner, and without W permissions, so attackers can not inject backdors and malware.
|
||||
|
||||
* resource min max allocation with validation
|
||||
|
|
@ -81,24 +69,14 @@
|
|||
|
||||
* Directory Protection on webapp and use webapp path as base path (validate)
|
||||
|
||||
* validate systemuser.home on server-side
|
||||
|
||||
* webapp backend option compatibility check? raise exception, missconfigured error
|
||||
|
||||
* admin systemuser home/directory, add default home and empty directory with has_shell on admin
|
||||
|
||||
* Resource used_list_display=True, allocated_list_displat=True, allow resources to show up on list_display
|
||||
|
||||
* BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when)
|
||||
|
||||
* Periodic task for cleaning old monitoring data
|
||||
|
||||
* Create an admin service_view with icons (like SaaS app)
|
||||
|
||||
* Resource graph for each related object
|
||||
|
||||
* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS, proxy model?
|
||||
|
||||
* prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org
|
||||
|
||||
* forms autocomplete="off", doesn't work in chrome
|
||||
|
|
@ -125,8 +103,6 @@ Php binaries should have this format: /usr/bin/php5.2-cgi
|
|||
|
||||
* contact.alternative_phone on a phone.tooltip, email:to
|
||||
|
||||
* better validate options and directives (url locations, filesystem paths, etc..)
|
||||
|
||||
* make sure that you understand the risks
|
||||
|
||||
* full support for deactivation of services/accounts
|
||||
|
|
@ -295,7 +271,7 @@ https://code.djangoproject.com/ticket/24576
|
|||
# Settings.parser.changes: if setting.value == default. remove
|
||||
# reload generic admin view ?redirect=http...
|
||||
# inspecting django db connection for asserting db readines? or performing a query
|
||||
# wake up django mailer on send_mail
|
||||
* wake up django mailer on send_mail
|
||||
|
||||
from orchestra.contrib.tasks import task
|
||||
import time, sys
|
||||
|
|
@ -308,14 +284,14 @@ https://code.djangoproject.com/ticket/24576
|
|||
time.sleep(1)
|
||||
counter.apply_async(10, '/tmp/kakas')
|
||||
|
||||
# Provide some fixtures with mocked data
|
||||
* Provide some fixtures with mocked data
|
||||
|
||||
|
||||
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
|
||||
TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
|
||||
TODO mount the filesystem with "nosuid" option
|
||||
|
||||
# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
|
||||
* uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
|
||||
|
||||
# mailboxes.address settings multiple local domains, not only one?
|
||||
# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
|
||||
|
|
@ -323,7 +299,7 @@ TODO mount the filesystem with "nosuid" option
|
|||
# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
|
||||
|
||||
# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
|
||||
# virtdomains file is not ideal, prevent fake/error on domains there! and make sure to chekc if this file is required!
|
||||
# virtdomains file is not ideal, prevent user provided fake/error domains there! and make sure to chekc if this file is required!
|
||||
|
||||
# Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
|
||||
orchestra-beat support for uwsgi cron
|
||||
|
|
@ -338,11 +314,9 @@ resorce monitoring more efficient, less mem an better queries for calc current d
|
|||
|
||||
# bill this https://orchestra.pangea.org/admin/orders/order/8236/ should be already billed, <= vs <
|
||||
# Convert rating method from function to PluginClass
|
||||
# Tests can not run because django.db.utils.ProgrammingError: relation "accounts_account" does not exist
|
||||
|
||||
# autoresponses on mailboxes, not addresses or remove them
|
||||
|
||||
# ACL don't give exec permissions to files!
|
||||
# force save and continue on routes (and others?)
|
||||
# gevent for python3
|
||||
apt-get install cython3
|
||||
|
|
@ -384,8 +358,6 @@ uwsgi --reload /tmp/project-master.pid
|
|||
# or if uwsgi was started with touch-reload=/tmp/somefile
|
||||
touch /tmp/somefile
|
||||
|
||||
# datetime metric storage granularity: otherwise innacurate detection of billed metric on order.billed_on
|
||||
|
||||
# Serializers.validation migration to DRF3: grep -r 'attrs, source' *|grep -v '~'
|
||||
serailzer self.instance on create.
|
||||
|
||||
|
|
@ -427,14 +399,14 @@ Case
|
|||
|
||||
# round decimals on every billing operation
|
||||
|
||||
# Add SPF record type
|
||||
|
||||
# OVZ TRAFFIC ACCOUNTING!!
|
||||
|
||||
# PHPlist cron, bounces and traffic (maybe specific mail script with sitename)
|
||||
|
||||
|
||||
# use "su $user --shell /bin/bash" on backends for security : MKDIR -p...
|
||||
|
||||
|
||||
# model.field.flatchoices
|
||||
|
||||
* This is beta software, please test thoroughly before putting into production and report back any issues.
|
||||
|
||||
# messages SMTP errors: temporary->deferre else Failed
|
||||
|
||||
# Don't enforce one contact per account? remove account.email in favour of contacts?
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ def fire_pending_messages(settings, db):
|
|||
for num, seconds in enumerate(MAILER_DEFERE_SECONDS):
|
||||
delta = timedelta(seconds=seconds)
|
||||
epoch = now-delta
|
||||
query_or.append("""(mailer_message.retries = %i AND mailer_message.last_retry <= '%s')"""
|
||||
query_or.append("""(mailer_message.retries = %i AND mailer_message.last_try <= '%s')"""
|
||||
% (num, epoch.isoformat().replace('T', ' ')))
|
||||
query = """\
|
||||
SELECT 1 FROM mailer_message
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
from django import forms
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from orchestra.core.validators import validate_password
|
||||
from orchestra.forms.widgets import SpanWidget
|
||||
from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
|
||||
|
||||
|
||||
class CleanAddressMixin(object):
|
||||
|
|
@ -15,25 +13,9 @@ class CleanAddressMixin(object):
|
|||
return domain
|
||||
|
||||
|
||||
class ListCreationForm(CleanAddressMixin, forms.ModelForm):
|
||||
password1 = forms.CharField(label=_("Password"), validators=[validate_password],
|
||||
widget=forms.PasswordInput)
|
||||
password2 = forms.CharField(label=_("Password confirmation"),
|
||||
widget=forms.PasswordInput,
|
||||
help_text=_("Enter the same password as above, for verification."))
|
||||
|
||||
def clean_password2(self):
|
||||
password1 = self.cleaned_data.get("password1")
|
||||
password2 = self.cleaned_data.get("password2")
|
||||
if password1 and password2 and password1 != password2:
|
||||
msg = _("The two password fields didn't match.")
|
||||
raise forms.ValidationError(msg)
|
||||
return password2
|
||||
class ListCreationForm(CleanAddressMixin, UserCreationForm):
|
||||
pass
|
||||
|
||||
|
||||
class ListChangeForm(CleanAddressMixin, forms.ModelForm):
|
||||
password = forms.CharField(label=_("Password"), required=False,
|
||||
widget=SpanWidget(display='<strong>Unknown password</strong>'),
|
||||
help_text=_("List passwords are not stored, so there is no way to see this "
|
||||
"list's password, but you can change the password using "
|
||||
"<a href=\"password/\">this form</a>."))
|
||||
class ListChangeForm(CleanAddressMixin, NonStoredUserChangeForm):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -28,13 +28,13 @@ COLORS = {
|
|||
class MessageAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
'display_subject', 'colored_state', 'priority', 'to_address', 'from_address',
|
||||
'created_at_delta', 'retries', 'last_retry_delta', 'num_logs',
|
||||
'created_at_delta', 'display_retries', 'last_try_delta',
|
||||
)
|
||||
list_filter = ('state', 'priority', 'retries')
|
||||
list_prefetch_related = ('logs__id')
|
||||
fieldsets = (
|
||||
(None, {
|
||||
'fields': ('state', 'priority', ('retries', 'last_retry_delta', 'created_at_delta'),
|
||||
'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'),
|
||||
'display_full_subject', 'display_from', 'display_to',
|
||||
'display_content'),
|
||||
}),
|
||||
|
|
@ -44,36 +44,36 @@ class MessageAdmin(admin.ModelAdmin):
|
|||
}),
|
||||
)
|
||||
readonly_fields = (
|
||||
'retries', 'last_retry_delta', 'created_at_delta', 'display_full_subject',
|
||||
'retries', 'last_try_delta', 'created_at_delta', 'display_full_subject',
|
||||
'display_to', 'display_from', 'display_content',
|
||||
)
|
||||
date_hierarchy = 'created_at'
|
||||
|
||||
colored_state = admin_colored('state', colors=COLORS)
|
||||
created_at_delta = admin_date('created_at')
|
||||
last_retry_delta = admin_date('last_retry')
|
||||
last_try_delta = admin_date('last_try')
|
||||
|
||||
def display_subject(self, instance):
|
||||
subject = instance.subject
|
||||
if len(subject) > 32:
|
||||
return subject[:32] + '…'
|
||||
if len(subject) > 64:
|
||||
return subject[:64] + '…'
|
||||
return subject
|
||||
display_subject.short_description = _("Subject")
|
||||
display_subject.admin_order_field = 'subject'
|
||||
display_subject.allow_tags = True
|
||||
|
||||
def num_logs(self, instance):
|
||||
num = instance.logs__count
|
||||
if num == 1:
|
||||
def display_retries(self, instance):
|
||||
num_logs = instance.logs__count
|
||||
if num_logs == 1:
|
||||
pk = instance.logs.all()[0].id
|
||||
url = reverse('admin:mailer_smtplog_change', args=(pk,))
|
||||
else:
|
||||
url = reverse('admin:mailer_smtplog_changelist')
|
||||
url += '?&message=%i' % instance.pk
|
||||
return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, num)
|
||||
num_logs.short_description = _("Logs")
|
||||
num_logs.admin_order_field = 'logs__count'
|
||||
num_logs.allow_tags = True
|
||||
return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, instance.retries)
|
||||
display_retries.short_description = _("Retries")
|
||||
display_retries.admin_order_field = 'retries'
|
||||
display_retries.allow_tags = True
|
||||
|
||||
def display_content(self, instance):
|
||||
part = email.message_from_string(instance.content)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,12 @@ def send_message(message, num=0, connection=None, bulk=settings.MAILER_BULK_MESS
|
|||
connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend')
|
||||
connection.open()
|
||||
error = None
|
||||
message.last_try = timezone.now()
|
||||
update_fields = ['last_try']
|
||||
if message.state != message.QUEUED:
|
||||
message.retries += 1
|
||||
update_fields.append('retries')
|
||||
message.save(update_fields=update_fields)
|
||||
try:
|
||||
connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content))
|
||||
except (SocketError,
|
||||
|
|
@ -49,7 +55,7 @@ def send_pending(bulk=settings.MAILER_BULK_MESSAGES):
|
|||
qs = Q()
|
||||
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
|
||||
delta = timedelta(seconds=seconds)
|
||||
qs = qs | Q(retries=retries, last_retry__lte=now-delta)
|
||||
qs = qs | Q(retries=retries, last_try__lte=now-delta)
|
||||
for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'):
|
||||
connection = send_message(message, num, connection, bulk)
|
||||
num += 1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('mailer', '0003_auto_20150617_1024'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='message',
|
||||
old_name='last_retry',
|
||||
new_name='last_try',
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='message',
|
||||
name='state',
|
||||
field=models.CharField(verbose_name='State', max_length=16, choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], default='QUEUED'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from . import settings
|
||||
|
|
@ -14,7 +13,7 @@ class Message(models.Model):
|
|||
(QUEUED, _("Queued")),
|
||||
(SENT, _("Sent")),
|
||||
(DEFERRED, _("Deferred")),
|
||||
(FAILED, _("Failes")),
|
||||
(FAILED, _("Failed")),
|
||||
)
|
||||
|
||||
CRITICAL = 0
|
||||
|
|
@ -36,8 +35,7 @@ class Message(models.Model):
|
|||
content = models.TextField(_("content"))
|
||||
created_at = models.DateTimeField(_("created at"), auto_now_add=True)
|
||||
retries = models.PositiveIntegerField(_("retries"), default=0)
|
||||
# TODO rename to last_try
|
||||
last_retry = models.DateTimeField(_("last try"), null=True)
|
||||
last_try = models.DateTimeField(_("last try"), null=True)
|
||||
|
||||
def __str__(self):
|
||||
return '%s to %s' % (self.subject, self.to_address)
|
||||
|
|
@ -47,9 +45,7 @@ class Message(models.Model):
|
|||
# Max tries
|
||||
if self.retries >= len(settings.MAILER_DEFERE_SECONDS):
|
||||
self.state = self.FAILED
|
||||
self.retries += 1
|
||||
self.last_retry = timezone.now()
|
||||
self.save(update_fields=('state', 'retries', 'last_retry'))
|
||||
self.save(update_fields=('state',))
|
||||
|
||||
def sent(self):
|
||||
self.state = self.SENT
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ class MonitorDataAdmin(ExtendedModelAdmin):
|
|||
list_display = ('id', 'monitor', content_object_link, 'display_created', 'value')
|
||||
list_filter = ('monitor', ResourceDataListFilter)
|
||||
add_fields = ('monitor', 'content_type', 'object_id', 'created_at', 'value')
|
||||
fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value')
|
||||
fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value', 'state')
|
||||
change_readonly_fields = fields
|
||||
list_select_related = ('content_type',)
|
||||
search_fields = ('content_object_repr',)
|
||||
|
|
@ -258,6 +258,7 @@ def resource_inline_factory(resources):
|
|||
if resource not in queryset_resources:
|
||||
kwargs = {
|
||||
'content_object': self.instance,
|
||||
'content_object_repr': str(self.instance),
|
||||
}
|
||||
if resource.default_allocation:
|
||||
kwargs['allocated'] = resource.default_allocation
|
||||
|
|
@ -323,7 +324,9 @@ def resource_inline_factory(resources):
|
|||
used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,))
|
||||
used = '<a href="%s">%s %s</a>' % (used_url, rdata.used, rdata.unit)
|
||||
return ' '.join(map(str, (used, update, history)))
|
||||
return _("Unknonw %s %s") % (update, history)
|
||||
if rdata.resource.monitors:
|
||||
return _("Unknonw %s %s") % (update, history)
|
||||
return _("No monitor")
|
||||
display_used.short_description = _("Used")
|
||||
display_used.allow_tags = True
|
||||
|
||||
|
|
|
|||
|
|
@ -57,8 +57,14 @@ class ServiceMonitor(ServiceBackend):
|
|||
return data.created_at
|
||||
|
||||
def process(self, line):
|
||||
""" line -> object_id, value """
|
||||
return line.split()
|
||||
""" line -> object_id, value, state"""
|
||||
result = line.split()
|
||||
if len(result) != 2:
|
||||
cls_name = self.__class__.__name__
|
||||
raise ValueError("%s expected '<id> <value>' got '%s'" % (cls_name, line))
|
||||
# State is None, unless your monitor needs to keep track of it
|
||||
result.append(None)
|
||||
return result
|
||||
|
||||
def store(self, log):
|
||||
""" stores monitored values from stdout """
|
||||
|
|
@ -68,16 +74,14 @@ class ServiceMonitor(ServiceBackend):
|
|||
ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
|
||||
for line in log.stdout.splitlines():
|
||||
line = line.strip()
|
||||
try:
|
||||
object_id, value = self.process(line)
|
||||
except ValueError:
|
||||
cls_name = self.__class__.__name__
|
||||
raise ValueError("%s expected '<id> <value>' got '%s'" % (cls_name, line))
|
||||
object_id, value, state = self.process(line)
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode('ascii')
|
||||
if isinstance(state, bytes):
|
||||
state = state.decode('ascii')
|
||||
content_object = ct.get_object_for_this_type(pk=object_id)
|
||||
MonitorData.objects.create(
|
||||
monitor=name, object_id=object_id, content_type=ct, value=value,
|
||||
monitor=name, object_id=object_id, content_type=ct, value=value, state=state,
|
||||
created_at=self.current_date, content_object_repr=str(content_object),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('resources', '0007_auto_20150723_1251'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='monitordata',
|
||||
name='state',
|
||||
field=models.DecimalField(verbose_name='state', null=True, max_digits=16, help_text='Optional field used to store current state needed for diff-based monitoring.', decimal_places=2),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('resources', '0008_monitordata_state'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='resourcedata',
|
||||
name='allocated',
|
||||
field=models.PositiveIntegerField(verbose_name='allocated', null=True, blank=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -172,8 +172,7 @@ class ResourceData(models.Model):
|
|||
used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True,
|
||||
editable=False)
|
||||
updated_at = models.DateTimeField(_("updated"), null=True, editable=False)
|
||||
allocated = models.DecimalField(_("allocated"), max_digits=8, decimal_places=2,
|
||||
null=True, blank=True)
|
||||
allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True)
|
||||
content_object_repr = models.CharField(_("content object representation"), max_length=256,
|
||||
editable=False)
|
||||
|
||||
|
|
@ -268,6 +267,8 @@ class MonitorData(models.Model):
|
|||
object_id = models.PositiveIntegerField(_("object id"), db_index=True)
|
||||
created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True)
|
||||
value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
|
||||
state = models.DecimalField(_("state"), max_digits=16, decimal_places=2, null=True,
|
||||
help_text=_("Optional field used to store current state needed for diff-based monitoring."))
|
||||
content_object_repr = models.CharField(_("content object representation"), max_length=256,
|
||||
editable=False)
|
||||
|
||||
|
|
@ -308,6 +309,7 @@ def create_resource_relation():
|
|||
)
|
||||
rdata = ResourceData(
|
||||
content_object=self.obj,
|
||||
content_object_repr=str(self.obj),
|
||||
resource=resource,
|
||||
allocated=resource.default_allocation
|
||||
)
|
||||
|
|
|
|||
|
|
@ -33,13 +33,18 @@ class ResourceSerializer(serializers.ModelSerializer):
|
|||
def insert_resource_serializers():
|
||||
# clean previous state
|
||||
for related in Resource._related:
|
||||
viewset = router.get_viewset(related)
|
||||
fields = list(viewset.serializer_class.Meta.fields)
|
||||
try:
|
||||
fields.remove('resources')
|
||||
except ValueError:
|
||||
viewset = router.get_viewset(related)
|
||||
except KeyError:
|
||||
# API viewset not registered
|
||||
pass
|
||||
viewset.serializer_class.Meta.fields = fields
|
||||
else:
|
||||
fields = list(viewset.serializer_class.Meta.fields)
|
||||
try:
|
||||
fields.remove('resources')
|
||||
except ValueError:
|
||||
pass
|
||||
viewset.serializer_class.Meta.fields = fields
|
||||
# Create nested serializers on target models
|
||||
for ct, resources in Resource.objects.group_by('content_type').items():
|
||||
model = ct.model_class()
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ def monitor(resource_id, ids=None):
|
|||
def cleanup_old_monitors(queryset=None):
|
||||
if queryset is None:
|
||||
from .models import MonitorData
|
||||
queryset = MonitorData.objects.filter()
|
||||
queryset = MonitorData.objects.all()
|
||||
delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS)
|
||||
threshold = timezone.now() - delta
|
||||
queryset = queryset.filter(created_at__lt=threshold)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,8 @@ class PhpListSaaSBackend(ServiceController):
|
|||
sys.stderr.write(msg + '\n')
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def _save(self, saas, server):
|
||||
def _install_or_change_password(self, saas, server):
|
||||
""" configures the database for the new site through HTTP to /admin/ """
|
||||
admin_link = 'https://%s/admin/' % saas.get_site_domain()
|
||||
sys.stdout.write('admin_link: %s\n' % admin_link)
|
||||
admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
|
||||
|
|
@ -75,7 +76,7 @@ class PhpListSaaSBackend(ServiceController):
|
|||
|
||||
def save(self, saas):
|
||||
if hasattr(saas, 'password'):
|
||||
self.append(self._save, saas)
|
||||
self.append(self._install_or_change_password, saas)
|
||||
context = self.get_context(saas)
|
||||
if context['crontab']:
|
||||
context['escaped_crontab'] = context['crontab'].replace('$', '\\$')
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@
|
|||
is_fee=&
|
||||
order_description=&
|
||||
ignore_period=TEN_DAYS&
|
||||
metric=max(logsteps(mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
|
||||
metric=max((mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
|
||||
nominal_price=20.00&
|
||||
tax=21&
|
||||
pricing_period=&
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import fnmatch
|
||||
import os
|
||||
import textwrap
|
||||
|
||||
|
|
@ -97,6 +98,51 @@ class UNIXUserBackend(ServiceController):
|
|||
else:
|
||||
self.append("rm -fr %(base_home)s" % context)
|
||||
|
||||
def grant_permissions(self, user, context):
|
||||
context['perms'] = user.set_perm_perms
|
||||
# Capital X adds execution permissions for directories, not files
|
||||
context['perms_X'] = context['perms'] + 'X'
|
||||
self.append(textwrap.dedent("""\
|
||||
# Grant execution permissions to every parent directory
|
||||
for access_path in %(access_paths)s; do
|
||||
# Preserve existing ACLs
|
||||
acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
|
||||
perms=$(echo "$acl" | cut -d':' -f3)
|
||||
perms=$(echo "$perms" | cut -c 1,2)x
|
||||
setfacl -m u:%(user)s:$perms "$access_path"
|
||||
} || setfacl -m u:%(user)s:--x "$access_path"
|
||||
done
|
||||
# Grant perms to existing files, excluding execution
|
||||
find '%(perm_to)s' -type f %(exclude_acl)s \\
|
||||
-exec setfacl -m u:%(user)s:%(perms)s {} \\;
|
||||
# Grant perms to extisting directories and set defaults for future content
|
||||
find '%(perm_to)s' -type d %(exclude_acl)s \\
|
||||
-exec setfacl -m u:%(user)s:%(perms_X)s -m d:u:%(user)s:%(perms_X)s {} \\;
|
||||
# Account group as the owner of new files
|
||||
chmod g+s '%(perm_to)s'""") % context
|
||||
)
|
||||
if not user.is_main:
|
||||
self.append(textwrap.dedent("""\
|
||||
# Grant access to main user
|
||||
find '%(perm_to)s' -type d %(exclude_acl)s \\
|
||||
-exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
|
||||
""") % context
|
||||
)
|
||||
|
||||
def revoke_permissions(self, context):
|
||||
revoke_perms = {
|
||||
'rw': '',
|
||||
'r': 'w',
|
||||
'w': 'r',
|
||||
}
|
||||
context['perms'] = revoke_perms[user.set_perm_perms]
|
||||
self.append(textwrap.dedent("""\
|
||||
# Revoke permissions
|
||||
find '%(perm_to)s' %(exclude_acl)s \\
|
||||
-exec setfacl -m u:%(user)s:%(perms)s {} \\;\
|
||||
""") % context
|
||||
)
|
||||
|
||||
def set_permission(self, user):
|
||||
context = self.get_context(user)
|
||||
context.update({
|
||||
|
|
@ -108,17 +154,10 @@ class UNIXUserBackend(ServiceController):
|
|||
context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude)
|
||||
exclude_acl.append('-not -path "%(exclude_acl)s"' % context)
|
||||
context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else ''
|
||||
if user.set_perm_perms == 'rw':
|
||||
context['perm_perms'] = 'rwx' if user.set_perm_action == 'grant' else '---'
|
||||
elif user.set_perm_perms == 'r':
|
||||
context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx'
|
||||
elif user.set_perm_perms == 'w':
|
||||
context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x'
|
||||
# Access paths
|
||||
head = user.set_perm_base_home
|
||||
relative = ''
|
||||
access_paths = ["'%s'" % head]
|
||||
import fnmatch
|
||||
for tail in user.set_perm_home_extension.split(os.sep)[:-1]:
|
||||
relative = os.path.join(relative, tail)
|
||||
for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS:
|
||||
|
|
@ -129,39 +168,11 @@ class UNIXUserBackend(ServiceController):
|
|||
head = os.path.join(head, tail)
|
||||
access_paths.append("'%s'" % head)
|
||||
context['access_paths'] = ' '.join(access_paths)
|
||||
|
||||
if user.set_perm_action == 'grant':
|
||||
self.append(textwrap.dedent("""\
|
||||
# Grant execution permissions to every parent directory
|
||||
for access_path in %(access_paths)s; do
|
||||
# Preserve existing ACLs
|
||||
acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
|
||||
perms=$(echo "$acl" | cut -d':' -f3)
|
||||
perms=$(echo "$perms" | cut -c 1,2)x
|
||||
setfacl -m u:%(user)s:$perms "$access_path"
|
||||
} || setfacl -m u:%(user)s:--x "$access_path"
|
||||
done
|
||||
# Grant perms to existing and future files
|
||||
find '%(perm_to)s' %(exclude_acl)s \\
|
||||
-exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;
|
||||
find '%(perm_to)s' -type d %(exclude_acl)s \\
|
||||
-exec setfacl -m d:u:%(user)s:%(perm_perms)s {} \\;
|
||||
# Account group as the owner of new files
|
||||
chmod g+s '%(perm_to)s'""") % context
|
||||
)
|
||||
if not user.is_main:
|
||||
self.append(textwrap.dedent("""\
|
||||
# Grant access to main user
|
||||
find '%(perm_to)s' -type d %(exclude_acl)s \\
|
||||
-exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
|
||||
""") % context
|
||||
)
|
||||
self.grant_permissions(user, context)
|
||||
elif user.set_perm_action == 'revoke':
|
||||
self.append(textwrap.dedent("""\
|
||||
# Revoke permissions
|
||||
find '%(perm_to)s' %(exclude_acl)s \\
|
||||
-exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\
|
||||
""") % context
|
||||
)
|
||||
self.revoke_permissions(user, context)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ class SystemUser(models.Model):
|
|||
help_text=_("Optional directory relative to user's home."))
|
||||
shell = models.CharField(_("shell"), max_length=32, choices=settings.SYSTEMUSERS_SHELLS,
|
||||
default=settings.SYSTEMUSERS_DEFAULT_SHELL)
|
||||
groups = models.ManyToManyField('self', blank=True, symmetrical=False,
|
||||
groups = models.ManyToManyField('self', blank=True, symmetrical=False,
|
||||
help_text=_("A new group will be created for the user. "
|
||||
"Which additional groups would you like them to be a member of?"))
|
||||
is_active = models.BooleanField(_("active"), default=True,
|
||||
|
|
|
|||
|
|
@ -6,18 +6,18 @@ from django.utils.translation import ugettext_lazy as _
|
|||
from orchestra.admin import ExtendedModelAdmin
|
||||
from orchestra.contrib.accounts.actions import list_accounts
|
||||
from orchestra.contrib.accounts.admin import AccountAdminMixin
|
||||
from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
|
||||
|
||||
from .forms import VPSChangeForm, VPSCreationForm
|
||||
from .models import VPS
|
||||
|
||||
|
||||
class VPSAdmin(AccountAdminMixin, ExtendedModelAdmin):
|
||||
list_display = ('hostname', 'type', 'template', 'account_link')
|
||||
list_filter = ('type', 'template')
|
||||
form = VPSChangeForm
|
||||
add_form = VPSCreationForm
|
||||
form = NonStoredUserChangeForm
|
||||
add_form = UserCreationForm
|
||||
readonly_fields = ('account_link',)
|
||||
change_readonly_fields = ('account', 'name', 'type', 'template')
|
||||
change_readonly_fields = ('account', 'hostname', 'type', 'template')
|
||||
fieldsets = (
|
||||
(None, {
|
||||
'classes': ('wide',),
|
||||
|
|
|
|||
|
|
@ -1,38 +1,114 @@
|
|||
from orchestra.contrib.orchestration import replace
|
||||
import decimal
|
||||
import textwrap
|
||||
|
||||
from orchestra.contrib.orchestration import ServiceController
|
||||
from orchestra.contrib.resources import ServiceMonitor
|
||||
|
||||
|
||||
class ProxmoxOVZ(ServiceController):
|
||||
model = 'vps.VPS'
|
||||
|
||||
RESOURCES = (
|
||||
('memory', 'mem'),
|
||||
('swap', 'swap'),
|
||||
('disk', 'disk')
|
||||
)
|
||||
GET_PROXMOX_INFO = textwrap.dedent("""
|
||||
function get_vz_info () {
|
||||
hostname=$1
|
||||
version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1)
|
||||
if [[ $version -lt 2 ]]; then
|
||||
conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:")
|
||||
CID=$(echo "$conf" | head -n1 | cut -d':' -f2)
|
||||
CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1)
|
||||
node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'})
|
||||
else
|
||||
conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf)
|
||||
node=$(echo "${conf}" | cut -d"/" -f5)
|
||||
CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1)
|
||||
fi
|
||||
echo $CTID $node
|
||||
}""")
|
||||
|
||||
def prepare(self):
|
||||
super(ProxmoxOVZ, self).prepare()
|
||||
self.append(self.GET_PROXMOX_INFO)
|
||||
|
||||
def get_vzset_args(self, vps):
|
||||
args = []
|
||||
for resource, arg_name in self.RESOURCES:
|
||||
try:
|
||||
allocation = getattr(vps.resources, resource).allocated
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
args.append('--%s %i' % (arg_name, allocation))
|
||||
return ' '.join(args)
|
||||
|
||||
def save(self, vps):
|
||||
context = self.get_context(vps)
|
||||
self.append('info=( $(get_vz_info %(hostname)s) )' % context)
|
||||
vzset_args = self.get_vzset_args(vps)
|
||||
if vzset_args:
|
||||
context['vzset_args'] = vzset_args
|
||||
self.append(textwrap.dedent("""\
|
||||
cat << EOF | ssh root@${info[1]}
|
||||
pvectl vzset ${info[0]} %(vzset_args)s
|
||||
EOF""") % context
|
||||
)
|
||||
if hasattr(vps, 'password'):
|
||||
context['password'] = vps.password.replace('$', '\\$')
|
||||
self.append(textwrap.dedent("""\
|
||||
cat << EOF | ssh root@${info[1]}
|
||||
echo 'root:%(password)s' | vzctl exec ${info[0]} chpasswd -e
|
||||
EOF""") % context
|
||||
)
|
||||
def get_context(self, vps):
|
||||
return {
|
||||
'hostname': vps.hostname,
|
||||
}
|
||||
|
||||
|
||||
# TODO rename to proxmox
|
||||
class OpenVZTraffic(ServiceMonitor):
|
||||
"""
|
||||
WARNING: Not fully implemeted
|
||||
"""
|
||||
model = 'vps.VPS'
|
||||
resource = ServiceMonitor.TRAFFIC
|
||||
monthly_sum_old_values = True
|
||||
|
||||
def prepare(self):
|
||||
super(OpenVZTraffic, self).prepare()
|
||||
self.append(ProxmoxOVZ.GET_PROXMOX_INFO)
|
||||
self.append(textwrap.dedent("""
|
||||
function monitor () {
|
||||
object_id=$1
|
||||
hostname=$2
|
||||
info=( $(get_vz_info $hostname) )
|
||||
cat << EOF | ssh root@${info[1]}
|
||||
vzctl exec ${info[0]} cat /proc/net/dev \\
|
||||
| grep venet0 \\
|
||||
| tr ':' ' ' \\
|
||||
| awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}'
|
||||
EOF
|
||||
}
|
||||
""")
|
||||
)
|
||||
|
||||
def process(self, line):
|
||||
""" diff with last stored value """
|
||||
object_id, value = line.split()
|
||||
""" diff with last stored state """
|
||||
object_id, value, state = super(OpenVZTraffic, self).process(line)
|
||||
value = decimal.Decimal(value)
|
||||
last = self.get_last_data(object_id)
|
||||
if not last or last.value > value:
|
||||
return object_id, value
|
||||
return object_id, value-last.value
|
||||
if not last or last.state > value:
|
||||
return object_id, value, value
|
||||
return object_id, value-last.state, value
|
||||
|
||||
def monitor(self, container):
|
||||
""" Get OpenVZ container traffic on a Proxmox +2.0 cluster """
|
||||
context = self.get_context(container)
|
||||
self.append(
|
||||
"CONF=$(grep -r 'HOSTNAME=\"%(hostname)s\"' /etc/pve/nodes/*/openvz/*.conf)" % context)
|
||||
self.append('NODE=$(echo "${CONF}" | cut -d"/" -f5)')
|
||||
self.append('CTID=$(echo "${CONF}" | cut -d"/" -f7 | cur -d"\." -f1)')
|
||||
self.append(
|
||||
"ssh root@${NODE} vzctl exec ${CTID} cat /proc/net/dev \\\n"
|
||||
" | grep venet0 \\\n"
|
||||
" | awk -F: '{print $2}' \\\n"
|
||||
" | awk '{print $1+$9}'")
|
||||
def monitor(self, vps):
|
||||
""" Get OpenVZ container traffic on a Proxmox cluster """
|
||||
context = self.get_context(vps)
|
||||
self.append('monitor %(object_id)s %(hostname)s' % context)
|
||||
|
||||
def get_context(self, container):
|
||||
context = {
|
||||
'hostname': container.hostname,
|
||||
def get_context(self, vps):
|
||||
return {
|
||||
'object_id': vps.id,
|
||||
'hostname': vps.hostname,
|
||||
}
|
||||
return replace(context, "'", '"')
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
from django import forms
|
||||
from django.contrib.auth.forms import ReadOnlyPasswordHashField
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
|
||||
class VPSCreationForm(forms.ModelForm):
|
||||
password1 = forms.CharField(label=_("Password"),
|
||||
widget=forms.PasswordInput)
|
||||
password2 = forms.CharField(label=_("Password confirmation"),
|
||||
widget=forms.PasswordInput,
|
||||
help_text=_("Enter the same password as above, for verification."))
|
||||
|
||||
class Meta:
|
||||
fields = ('username', 'account', 'type', 'template')
|
||||
|
||||
def clean_password2(self):
|
||||
password1 = self.cleaned_data.get("password1")
|
||||
password2 = self.cleaned_data.get("password2")
|
||||
if password1 and password2 and password1 != password2:
|
||||
msg = _("The two password fields didn't match.")
|
||||
raise forms.ValidationError(msg)
|
||||
return password2
|
||||
|
||||
def save(self, commit=True):
|
||||
vps = super(VPSCreationForm, self).save(commit=False)
|
||||
vps.set_password(self.cleaned_data["password1"])
|
||||
if commit:
|
||||
vps.save()
|
||||
return vps
|
||||
|
||||
|
||||
class VPSChangeForm(forms.ModelForm):
|
||||
password = ReadOnlyPasswordHashField(label=_("Password"),
|
||||
help_text=_("Raw passwords are not stored, so there is no way to see "
|
||||
"this user's password, but you can change the password "
|
||||
"using <a href=\"password/\">this form</a>."))
|
||||
|
||||
def clean_password(self):
|
||||
return self.initial["password"]
|
||||
23
orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py
Normal file
23
orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('vps', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='vps',
|
||||
name='password',
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='vps',
|
||||
name='template',
|
||||
field=models.CharField(default='debian7', verbose_name='template', choices=[('debian7', 'Debian 7 - Wheezy')], max_length=64, help_text='Initial template.'),
|
||||
),
|
||||
]
|
||||
|
|
@ -13,9 +13,8 @@ class VPS(models.Model):
|
|||
type = models.CharField(_("type"), max_length=64, choices=settings.VPS_TYPES,
|
||||
default=settings.VPS_DEFAULT_TYPE)
|
||||
template = models.CharField(_("template"), max_length=64,
|
||||
choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE)
|
||||
password = models.CharField(_('password'), max_length=128,
|
||||
help_text=_("<TT>root</TT> password of this virtual machine"))
|
||||
choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE,
|
||||
help_text=_("Initial template."))
|
||||
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
|
||||
related_name='vpss')
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ class UserCreationForm(forms.ModelForm):
|
|||
class UserChangeForm(forms.ModelForm):
|
||||
password = auth_forms.ReadOnlyPasswordHashField(label=_("Password"),
|
||||
help_text=_("Raw passwords are not stored, so there is no way to see "
|
||||
"this user's password, but you can change the password "
|
||||
"this user's password, but you can change it by "
|
||||
"using <a href=\"password/\">this form</a>."))
|
||||
|
||||
def clean_password(self):
|
||||
|
|
@ -70,6 +70,13 @@ class UserChangeForm(forms.ModelForm):
|
|||
return self.initial["password"]
|
||||
|
||||
|
||||
class NonStoredUserChangeForm(forms.ModelForm):
|
||||
password = forms.CharField(label=_("Password"), required=False,
|
||||
widget=SpanWidget(display='<strong>Unknown password</strong>'),
|
||||
help_text=_("This service's password is not stored, so there is no way to see it, "
|
||||
"but you can change it using <a href=\"password/\">this form</a>."))
|
||||
|
||||
|
||||
class ReadOnlyFormMixin(object):
|
||||
"""
|
||||
Mixin class for ModelForm or Form that provides support for SpanField on readonly fields
|
||||
|
|
|
|||
|
|
@ -142,7 +142,11 @@ def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', async
|
|||
|
||||
def sshrun(addr, command, *args, executable='bash', persist=False, **kwargs):
|
||||
from .. import settings
|
||||
options = ['stricthostkeychecking=no']
|
||||
options = [
|
||||
'stricthostkeychecking=no',
|
||||
'BatchMode=yes',
|
||||
'EscapeChar=none',
|
||||
]
|
||||
if persist:
|
||||
options.extend((
|
||||
'ControlMaster=auto',
|
||||
|
|
|
|||
Loading…
Reference in a new issue