diff --git a/TODO.md b/TODO.md
index c2727678..bada898b 100644
--- a/TODO.md
+++ b/TODO.md
@@ -36,8 +36,6 @@
* What fields we really need on contacts? name email phone and what more?
-* Redirect junk emails and delete every 30 days?
-
* DOC: Complitely decouples scripts execution, billing, service definition
* init.d celery scripts
@@ -58,21 +56,11 @@
* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture --keepdb
* ForeignKey.swappable
-* Field.editable
-* ManyToManyField.symmetrical = False (user group)
* REST PERMISSIONS
-* caching based on "def text2int(textnum, numwords={}):"
-
-* sync() ServiceController method that synchronizes orchestra and servers (delete or import)
-
-* consider removing mailbox support on forward (user@pangea.org instead)
-
* Databases.User add reverse M2M databases widget (like mailbox.addresses)
-* Grant permissions to systemusers
-
* Make one dedicated CGI user for each account only for CGI execution (fpm/fcgid). Different from the files owner, and without W permissions, so attackers can not inject backdors and malware.
* resource min max allocation with validation
@@ -81,24 +69,14 @@
* Directory Protection on webapp and use webapp path as base path (validate)
-* validate systemuser.home on server-side
-
* webapp backend option compatibility check? raise exception, missconfigured error
-* admin systemuser home/directory, add default home and empty directory with has_shell on admin
-
* Resource used_list_display=True, allocated_list_displat=True, allow resources to show up on list_display
* BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when)
-* Periodic task for cleaning old monitoring data
-
* Create an admin service_view with icons (like SaaS app)
-* Resource graph for each related object
-
-* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS, proxy model?
-
* prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org
* forms autocomplete="off", doesn't work in chrome
@@ -125,8 +103,6 @@ Php binaries should have this format: /usr/bin/php5.2-cgi
* contact.alternative_phone on a phone.tooltip, email:to
-* better validate options and directives (url locations, filesystem paths, etc..)
-
* make sure that you understand the risks
* full support for deactivation of services/accounts
@@ -295,7 +271,7 @@ https://code.djangoproject.com/ticket/24576
# Settings.parser.changes: if setting.value == default. remove
# reload generic admin view ?redirect=http...
# inspecting django db connection for asserting db readines? or performing a query
-# wake up django mailer on send_mail
+* wake up django mailer on send_mail
from orchestra.contrib.tasks import task
import time, sys
@@ -308,14 +284,14 @@ https://code.djangoproject.com/ticket/24576
time.sleep(1)
counter.apply_async(10, '/tmp/kakas')
-# Provide some fixtures with mocked data
+* Provide some fixtures with mocked data
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
TODO mount the filesystem with "nosuid" option
-# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
+* uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
# mailboxes.address settings multiple local domains, not only one?
# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
@@ -323,7 +299,7 @@ TODO mount the filesystem with "nosuid" option
# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
-# virtdomains file is not ideal, prevent fake/error on domains there! and make sure to chekc if this file is required!
+# virtdomains file is not ideal, prevent user provided fake/error domains there! and make sure to chekc if this file is required!
# Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
orchestra-beat support for uwsgi cron
@@ -338,11 +314,9 @@ resorce monitoring more efficient, less mem an better queries for calc current d
# bill this https://orchestra.pangea.org/admin/orders/order/8236/ should be already billed, <= vs <
# Convert rating method from function to PluginClass
-# Tests can not run because django.db.utils.ProgrammingError: relation "accounts_account" does not exist
# autoresponses on mailboxes, not addresses or remove them
-# ACL don't give exec permissions to files!
# force save and continue on routes (and others?)
# gevent for python3
apt-get install cython3
@@ -384,8 +358,6 @@ uwsgi --reload /tmp/project-master.pid
# or if uwsgi was started with touch-reload=/tmp/somefile
touch /tmp/somefile
-# datetime metric storage granularity: otherwise innacurate detection of billed metric on order.billed_on
-
# Serializers.validation migration to DRF3: grep -r 'attrs, source' *|grep -v '~'
serailzer self.instance on create.
@@ -427,14 +399,14 @@ Case
# round decimals on every billing operation
-# Add SPF record type
-
-# OVZ TRAFFIC ACCOUNTING!!
-
# PHPlist cron, bounces and traffic (maybe specific mail script with sitename)
-
# use "su $user --shell /bin/bash" on backends for security : MKDIR -p...
-
# model.field.flatchoices
+
+* This is beta software, please test thoroughly before putting into production and report back any issues.
+
+# messages SMTP errors: temporary->deferre else Failed
+
+# Don't enforce one contact per account? remove account.email in favour of contacts?
diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat
index 53c5175b..09d13fa2 100755
--- a/orchestra/bin/orchestra-beat
+++ b/orchestra/bin/orchestra-beat
@@ -191,7 +191,7 @@ def fire_pending_messages(settings, db):
for num, seconds in enumerate(MAILER_DEFERE_SECONDS):
delta = timedelta(seconds=seconds)
epoch = now-delta
- query_or.append("""(mailer_message.retries = %i AND mailer_message.last_retry <= '%s')"""
+ query_or.append("""(mailer_message.retries = %i AND mailer_message.last_try <= '%s')"""
% (num, epoch.isoformat().replace('T', ' ')))
query = """\
SELECT 1 FROM mailer_message
diff --git a/orchestra/contrib/lists/forms.py b/orchestra/contrib/lists/forms.py
index 0ffff1f7..c83b0f3a 100644
--- a/orchestra/contrib/lists/forms.py
+++ b/orchestra/contrib/lists/forms.py
@@ -1,8 +1,6 @@
-from django import forms
from django.utils.translation import ugettext_lazy as _
-from orchestra.core.validators import validate_password
-from orchestra.forms.widgets import SpanWidget
+from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
class CleanAddressMixin(object):
@@ -15,25 +13,9 @@ class CleanAddressMixin(object):
return domain
-class ListCreationForm(CleanAddressMixin, forms.ModelForm):
- password1 = forms.CharField(label=_("Password"), validators=[validate_password],
- widget=forms.PasswordInput)
- password2 = forms.CharField(label=_("Password confirmation"),
- widget=forms.PasswordInput,
- help_text=_("Enter the same password as above, for verification."))
-
- def clean_password2(self):
- password1 = self.cleaned_data.get("password1")
- password2 = self.cleaned_data.get("password2")
- if password1 and password2 and password1 != password2:
- msg = _("The two password fields didn't match.")
- raise forms.ValidationError(msg)
- return password2
-
+class ListCreationForm(CleanAddressMixin, UserCreationForm):
+ pass
-class ListChangeForm(CleanAddressMixin, forms.ModelForm):
- password = forms.CharField(label=_("Password"), required=False,
- widget=SpanWidget(display='Unknown password'),
- help_text=_("List passwords are not stored, so there is no way to see this "
- "list's password, but you can change the password using "
- "this form."))
+
+class ListChangeForm(CleanAddressMixin, NonStoredUserChangeForm):
+ pass
diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py
index b559d05e..6ee69e21 100644
--- a/orchestra/contrib/mailer/admin.py
+++ b/orchestra/contrib/mailer/admin.py
@@ -28,13 +28,13 @@ COLORS = {
class MessageAdmin(admin.ModelAdmin):
list_display = (
'display_subject', 'colored_state', 'priority', 'to_address', 'from_address',
- 'created_at_delta', 'retries', 'last_retry_delta', 'num_logs',
+ 'created_at_delta', 'display_retries', 'last_try_delta',
)
list_filter = ('state', 'priority', 'retries')
list_prefetch_related = ('logs__id')
fieldsets = (
(None, {
- 'fields': ('state', 'priority', ('retries', 'last_retry_delta', 'created_at_delta'),
+ 'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'),
'display_full_subject', 'display_from', 'display_to',
'display_content'),
}),
@@ -44,36 +44,36 @@ class MessageAdmin(admin.ModelAdmin):
}),
)
readonly_fields = (
- 'retries', 'last_retry_delta', 'created_at_delta', 'display_full_subject',
+ 'retries', 'last_try_delta', 'created_at_delta', 'display_full_subject',
'display_to', 'display_from', 'display_content',
)
date_hierarchy = 'created_at'
colored_state = admin_colored('state', colors=COLORS)
created_at_delta = admin_date('created_at')
- last_retry_delta = admin_date('last_retry')
+ last_try_delta = admin_date('last_try')
def display_subject(self, instance):
subject = instance.subject
- if len(subject) > 32:
- return subject[:32] + '…'
+ if len(subject) > 64:
+ return subject[:64] + '…'
return subject
display_subject.short_description = _("Subject")
display_subject.admin_order_field = 'subject'
display_subject.allow_tags = True
- def num_logs(self, instance):
- num = instance.logs__count
- if num == 1:
+ def display_retries(self, instance):
+ num_logs = instance.logs__count
+ if num_logs == 1:
pk = instance.logs.all()[0].id
url = reverse('admin:mailer_smtplog_change', args=(pk,))
else:
url = reverse('admin:mailer_smtplog_changelist')
url += '?&message=%i' % instance.pk
- return '%d' % (url, num)
- num_logs.short_description = _("Logs")
- num_logs.admin_order_field = 'logs__count'
- num_logs.allow_tags = True
+ return '%d' % (url, instance.retries)
+ display_retries.short_description = _("Retries")
+ display_retries.admin_order_field = 'retries'
+ display_retries.allow_tags = True
def display_content(self, instance):
part = email.message_from_string(instance.content)
diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py
index bbcbd3ed..ddf06c2f 100644
--- a/orchestra/contrib/mailer/engine.py
+++ b/orchestra/contrib/mailer/engine.py
@@ -23,6 +23,12 @@ def send_message(message, num=0, connection=None, bulk=settings.MAILER_BULK_MESS
connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend')
connection.open()
error = None
+ message.last_try = timezone.now()
+ update_fields = ['last_try']
+ if message.state != message.QUEUED:
+ message.retries += 1
+ update_fields.append('retries')
+ message.save(update_fields=update_fields)
try:
connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content))
except (SocketError,
@@ -49,7 +55,7 @@ def send_pending(bulk=settings.MAILER_BULK_MESSAGES):
qs = Q()
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
delta = timedelta(seconds=seconds)
- qs = qs | Q(retries=retries, last_retry__lte=now-delta)
+ qs = qs | Q(retries=retries, last_try__lte=now-delta)
for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'):
connection = send_message(message, num, connection, bulk)
num += 1
diff --git a/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py b/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py
new file mode 100644
index 00000000..d224585b
--- /dev/null
+++ b/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('mailer', '0003_auto_20150617_1024'),
+ ]
+
+ operations = [
+ migrations.RenameField(
+ model_name='message',
+ old_name='last_retry',
+ new_name='last_try',
+ ),
+ migrations.AlterField(
+ model_name='message',
+ name='state',
+ field=models.CharField(verbose_name='State', max_length=16, choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], default='QUEUED'),
+ ),
+ ]
diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py
index 4d396c13..519ecfa5 100644
--- a/orchestra/contrib/mailer/models.py
+++ b/orchestra/contrib/mailer/models.py
@@ -1,5 +1,4 @@
from django.db import models
-from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from . import settings
@@ -14,7 +13,7 @@ class Message(models.Model):
(QUEUED, _("Queued")),
(SENT, _("Sent")),
(DEFERRED, _("Deferred")),
- (FAILED, _("Failes")),
+ (FAILED, _("Failed")),
)
CRITICAL = 0
@@ -36,8 +35,7 @@ class Message(models.Model):
content = models.TextField(_("content"))
created_at = models.DateTimeField(_("created at"), auto_now_add=True)
retries = models.PositiveIntegerField(_("retries"), default=0)
- # TODO rename to last_try
- last_retry = models.DateTimeField(_("last try"), null=True)
+ last_try = models.DateTimeField(_("last try"), null=True)
def __str__(self):
return '%s to %s' % (self.subject, self.to_address)
@@ -47,9 +45,7 @@ class Message(models.Model):
# Max tries
if self.retries >= len(settings.MAILER_DEFERE_SECONDS):
self.state = self.FAILED
- self.retries += 1
- self.last_retry = timezone.now()
- self.save(update_fields=('state', 'retries', 'last_retry'))
+ self.save(update_fields=('state',))
def sent(self):
self.state = self.SENT
diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py
index 66062f31..56f8108f 100644
--- a/orchestra/contrib/resources/admin.py
+++ b/orchestra/contrib/resources/admin.py
@@ -198,7 +198,7 @@ class MonitorDataAdmin(ExtendedModelAdmin):
list_display = ('id', 'monitor', content_object_link, 'display_created', 'value')
list_filter = ('monitor', ResourceDataListFilter)
add_fields = ('monitor', 'content_type', 'object_id', 'created_at', 'value')
- fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value')
+ fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value', 'state')
change_readonly_fields = fields
list_select_related = ('content_type',)
search_fields = ('content_object_repr',)
@@ -258,6 +258,7 @@ def resource_inline_factory(resources):
if resource not in queryset_resources:
kwargs = {
'content_object': self.instance,
+ 'content_object_repr': str(self.instance),
}
if resource.default_allocation:
kwargs['allocated'] = resource.default_allocation
@@ -323,7 +324,9 @@ def resource_inline_factory(resources):
used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,))
used = '%s %s' % (used_url, rdata.used, rdata.unit)
return ' '.join(map(str, (used, update, history)))
- return _("Unknonw %s %s") % (update, history)
+ if rdata.resource.monitors:
+ return _("Unknonw %s %s") % (update, history)
+ return _("No monitor")
display_used.short_description = _("Used")
display_used.allow_tags = True
diff --git a/orchestra/contrib/resources/backends.py b/orchestra/contrib/resources/backends.py
index 2b6ca374..e12e6135 100644
--- a/orchestra/contrib/resources/backends.py
+++ b/orchestra/contrib/resources/backends.py
@@ -57,8 +57,14 @@ class ServiceMonitor(ServiceBackend):
return data.created_at
def process(self, line):
- """ line -> object_id, value """
- return line.split()
+ """ line -> object_id, value, state"""
+ result = line.split()
+ if len(result) != 2:
+ cls_name = self.__class__.__name__
+ raise ValueError("%s expected ' ' got '%s'" % (cls_name, line))
+ # State is None, unless your monitor needs to keep track of it
+ result.append(None)
+ return result
def store(self, log):
""" stores monitored values from stdout """
@@ -68,16 +74,14 @@ class ServiceMonitor(ServiceBackend):
ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
for line in log.stdout.splitlines():
line = line.strip()
- try:
- object_id, value = self.process(line)
- except ValueError:
- cls_name = self.__class__.__name__
- raise ValueError("%s expected ' ' got '%s'" % (cls_name, line))
+ object_id, value, state = self.process(line)
if isinstance(value, bytes):
value = value.decode('ascii')
+ if isinstance(state, bytes):
+ state = state.decode('ascii')
content_object = ct.get_object_for_this_type(pk=object_id)
MonitorData.objects.create(
- monitor=name, object_id=object_id, content_type=ct, value=value,
+ monitor=name, object_id=object_id, content_type=ct, value=value, state=state,
created_at=self.current_date, content_object_repr=str(content_object),
)
diff --git a/orchestra/contrib/resources/migrations/0008_monitordata_state.py b/orchestra/contrib/resources/migrations/0008_monitordata_state.py
new file mode 100644
index 00000000..65475d4e
--- /dev/null
+++ b/orchestra/contrib/resources/migrations/0008_monitordata_state.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resources', '0007_auto_20150723_1251'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='monitordata',
+ name='state',
+ field=models.DecimalField(verbose_name='state', null=True, max_digits=16, help_text='Optional field used to store current state needed for diff-based monitoring.', decimal_places=2),
+ ),
+ ]
diff --git a/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py b/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py
new file mode 100644
index 00000000..f93fb2c3
--- /dev/null
+++ b/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resources', '0008_monitordata_state'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='resourcedata',
+ name='allocated',
+ field=models.PositiveIntegerField(verbose_name='allocated', null=True, blank=True),
+ ),
+ ]
diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py
index b49497c7..24da00b5 100644
--- a/orchestra/contrib/resources/models.py
+++ b/orchestra/contrib/resources/models.py
@@ -172,8 +172,7 @@ class ResourceData(models.Model):
used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True,
editable=False)
updated_at = models.DateTimeField(_("updated"), null=True, editable=False)
- allocated = models.DecimalField(_("allocated"), max_digits=8, decimal_places=2,
- null=True, blank=True)
+ allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True)
content_object_repr = models.CharField(_("content object representation"), max_length=256,
editable=False)
@@ -268,6 +267,8 @@ class MonitorData(models.Model):
object_id = models.PositiveIntegerField(_("object id"), db_index=True)
created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True)
value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
+ state = models.DecimalField(_("state"), max_digits=16, decimal_places=2, null=True,
+ help_text=_("Optional field used to store current state needed for diff-based monitoring."))
content_object_repr = models.CharField(_("content object representation"), max_length=256,
editable=False)
@@ -308,6 +309,7 @@ def create_resource_relation():
)
rdata = ResourceData(
content_object=self.obj,
+ content_object_repr=str(self.obj),
resource=resource,
allocated=resource.default_allocation
)
diff --git a/orchestra/contrib/resources/serializers.py b/orchestra/contrib/resources/serializers.py
index 0ffadf2f..140674ed 100644
--- a/orchestra/contrib/resources/serializers.py
+++ b/orchestra/contrib/resources/serializers.py
@@ -33,13 +33,18 @@ class ResourceSerializer(serializers.ModelSerializer):
def insert_resource_serializers():
# clean previous state
for related in Resource._related:
- viewset = router.get_viewset(related)
- fields = list(viewset.serializer_class.Meta.fields)
try:
- fields.remove('resources')
- except ValueError:
+ viewset = router.get_viewset(related)
+ except KeyError:
+ # API viewset not registered
pass
- viewset.serializer_class.Meta.fields = fields
+ else:
+ fields = list(viewset.serializer_class.Meta.fields)
+ try:
+ fields.remove('resources')
+ except ValueError:
+ pass
+ viewset.serializer_class.Meta.fields = fields
# Create nested serializers on target models
for ct, resources in Resource.objects.group_by('content_type').items():
model = ct.model_class()
diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py
index 38ab8220..b802e876 100644
--- a/orchestra/contrib/resources/tasks.py
+++ b/orchestra/contrib/resources/tasks.py
@@ -63,7 +63,7 @@ def monitor(resource_id, ids=None):
def cleanup_old_monitors(queryset=None):
if queryset is None:
from .models import MonitorData
- queryset = MonitorData.objects.filter()
+ queryset = MonitorData.objects.all()
delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS)
threshold = timezone.now() - delta
queryset = queryset.filter(created_at__lt=threshold)
diff --git a/orchestra/contrib/saas/backends/phplist.py b/orchestra/contrib/saas/backends/phplist.py
index 667498a0..03bd5c0b 100644
--- a/orchestra/contrib/saas/backends/phplist.py
+++ b/orchestra/contrib/saas/backends/phplist.py
@@ -34,7 +34,8 @@ class PhpListSaaSBackend(ServiceController):
sys.stderr.write(msg + '\n')
raise RuntimeError(msg)
- def _save(self, saas, server):
+ def _install_or_change_password(self, saas, server):
+ """ configures the database for the new site through HTTP to /admin/ """
admin_link = 'https://%s/admin/' % saas.get_site_domain()
sys.stdout.write('admin_link: %s\n' % admin_link)
admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
@@ -75,7 +76,7 @@ class PhpListSaaSBackend(ServiceController):
def save(self, saas):
if hasattr(saas, 'password'):
- self.append(self._save, saas)
+ self.append(self._install_or_change_password, saas)
context = self.get_context(saas)
if context['crontab']:
context['escaped_crontab'] = context['crontab'].replace('$', '\\$')
diff --git a/orchestra/contrib/services/templates/admin/services/service/change_form.html b/orchestra/contrib/services/templates/admin/services/service/change_form.html
index 07642046..44f7c1ae 100644
--- a/orchestra/contrib/services/templates/admin/services/service/change_form.html
+++ b/orchestra/contrib/services/templates/admin/services/service/change_form.html
@@ -37,7 +37,7 @@
is_fee=&
order_description=&
ignore_period=TEN_DAYS&
- metric=max(logsteps(mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
+ metric=max((mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
nominal_price=20.00&
tax=21&
pricing_period=&
diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py
index c02b169c..673c42eb 100644
--- a/orchestra/contrib/systemusers/backends.py
+++ b/orchestra/contrib/systemusers/backends.py
@@ -1,3 +1,4 @@
+import fnmatch
import os
import textwrap
@@ -97,6 +98,51 @@ class UNIXUserBackend(ServiceController):
else:
self.append("rm -fr %(base_home)s" % context)
+ def grant_permissions(self, user, context):
+ context['perms'] = user.set_perm_perms
+ # Capital X adds execution permissions for directories, not files
+ context['perms_X'] = context['perms'] + 'X'
+ self.append(textwrap.dedent("""\
+ # Grant execution permissions to every parent directory
+ for access_path in %(access_paths)s; do
+ # Preserve existing ACLs
+ acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
+ perms=$(echo "$acl" | cut -d':' -f3)
+ perms=$(echo "$perms" | cut -c 1,2)x
+ setfacl -m u:%(user)s:$perms "$access_path"
+ } || setfacl -m u:%(user)s:--x "$access_path"
+ done
+ # Grant perms to existing files, excluding execution
+ find '%(perm_to)s' -type f %(exclude_acl)s \\
+ -exec setfacl -m u:%(user)s:%(perms)s {} \\;
+ # Grant perms to extisting directories and set defaults for future content
+ find '%(perm_to)s' -type d %(exclude_acl)s \\
+ -exec setfacl -m u:%(user)s:%(perms_X)s -m d:u:%(user)s:%(perms_X)s {} \\;
+ # Account group as the owner of new files
+ chmod g+s '%(perm_to)s'""") % context
+ )
+ if not user.is_main:
+ self.append(textwrap.dedent("""\
+ # Grant access to main user
+ find '%(perm_to)s' -type d %(exclude_acl)s \\
+ -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
+ """) % context
+ )
+
+ def revoke_permissions(self, context):
+ revoke_perms = {
+ 'rw': '',
+ 'r': 'w',
+ 'w': 'r',
+ }
+ context['perms'] = revoke_perms[user.set_perm_perms]
+ self.append(textwrap.dedent("""\
+ # Revoke permissions
+ find '%(perm_to)s' %(exclude_acl)s \\
+ -exec setfacl -m u:%(user)s:%(perms)s {} \\;\
+ """) % context
+ )
+
def set_permission(self, user):
context = self.get_context(user)
context.update({
@@ -108,17 +154,10 @@ class UNIXUserBackend(ServiceController):
context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude)
exclude_acl.append('-not -path "%(exclude_acl)s"' % context)
context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else ''
- if user.set_perm_perms == 'rw':
- context['perm_perms'] = 'rwx' if user.set_perm_action == 'grant' else '---'
- elif user.set_perm_perms == 'r':
- context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx'
- elif user.set_perm_perms == 'w':
- context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x'
# Access paths
head = user.set_perm_base_home
relative = ''
access_paths = ["'%s'" % head]
- import fnmatch
for tail in user.set_perm_home_extension.split(os.sep)[:-1]:
relative = os.path.join(relative, tail)
for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS:
@@ -129,39 +168,11 @@ class UNIXUserBackend(ServiceController):
head = os.path.join(head, tail)
access_paths.append("'%s'" % head)
context['access_paths'] = ' '.join(access_paths)
+
if user.set_perm_action == 'grant':
- self.append(textwrap.dedent("""\
- # Grant execution permissions to every parent directory
- for access_path in %(access_paths)s; do
- # Preserve existing ACLs
- acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
- perms=$(echo "$acl" | cut -d':' -f3)
- perms=$(echo "$perms" | cut -c 1,2)x
- setfacl -m u:%(user)s:$perms "$access_path"
- } || setfacl -m u:%(user)s:--x "$access_path"
- done
- # Grant perms to existing and future files
- find '%(perm_to)s' %(exclude_acl)s \\
- -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;
- find '%(perm_to)s' -type d %(exclude_acl)s \\
- -exec setfacl -m d:u:%(user)s:%(perm_perms)s {} \\;
- # Account group as the owner of new files
- chmod g+s '%(perm_to)s'""") % context
- )
- if not user.is_main:
- self.append(textwrap.dedent("""\
- # Grant access to main user
- find '%(perm_to)s' -type d %(exclude_acl)s \\
- -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
- """) % context
- )
+ self.grant_permissions(user, context)
elif user.set_perm_action == 'revoke':
- self.append(textwrap.dedent("""\
- # Revoke permissions
- find '%(perm_to)s' %(exclude_acl)s \\
- -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\
- """) % context
- )
+ self.revoke_permissions(user, context)
else:
raise NotImplementedError()
diff --git a/orchestra/contrib/systemusers/models.py b/orchestra/contrib/systemusers/models.py
index 62798d2c..06cb775d 100644
--- a/orchestra/contrib/systemusers/models.py
+++ b/orchestra/contrib/systemusers/models.py
@@ -45,7 +45,7 @@ class SystemUser(models.Model):
help_text=_("Optional directory relative to user's home."))
shell = models.CharField(_("shell"), max_length=32, choices=settings.SYSTEMUSERS_SHELLS,
default=settings.SYSTEMUSERS_DEFAULT_SHELL)
- groups = models.ManyToManyField('self', blank=True, symmetrical=False,
+ groups = models.ManyToManyField('self', blank=True, symmetrical=False,
help_text=_("A new group will be created for the user. "
"Which additional groups would you like them to be a member of?"))
is_active = models.BooleanField(_("active"), default=True,
diff --git a/orchestra/contrib/vps/admin.py b/orchestra/contrib/vps/admin.py
index d81c85d0..14624ba0 100644
--- a/orchestra/contrib/vps/admin.py
+++ b/orchestra/contrib/vps/admin.py
@@ -6,18 +6,18 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin
from orchestra.contrib.accounts.actions import list_accounts
from orchestra.contrib.accounts.admin import AccountAdminMixin
+from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
-from .forms import VPSChangeForm, VPSCreationForm
from .models import VPS
class VPSAdmin(AccountAdminMixin, ExtendedModelAdmin):
list_display = ('hostname', 'type', 'template', 'account_link')
list_filter = ('type', 'template')
- form = VPSChangeForm
- add_form = VPSCreationForm
+ form = NonStoredUserChangeForm
+ add_form = UserCreationForm
readonly_fields = ('account_link',)
- change_readonly_fields = ('account', 'name', 'type', 'template')
+ change_readonly_fields = ('account', 'hostname', 'type', 'template')
fieldsets = (
(None, {
'classes': ('wide',),
diff --git a/orchestra/contrib/vps/backends.py b/orchestra/contrib/vps/backends.py
index 1dd12dbb..f4fed982 100644
--- a/orchestra/contrib/vps/backends.py
+++ b/orchestra/contrib/vps/backends.py
@@ -1,38 +1,114 @@
-from orchestra.contrib.orchestration import replace
+import decimal
+import textwrap
+
+from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
+class ProxmoxOVZ(ServiceController):
+ model = 'vps.VPS'
+
+ RESOURCES = (
+ ('memory', 'mem'),
+ ('swap', 'swap'),
+ ('disk', 'disk')
+ )
+ GET_PROXMOX_INFO = textwrap.dedent("""
+ function get_vz_info () {
+ hostname=$1
+ version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1)
+ if [[ $version -lt 2 ]]; then
+ conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:")
+ CID=$(echo "$conf" | head -n1 | cut -d':' -f2)
+ CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1)
+ node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'})
+ else
+ conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf)
+ node=$(echo "${conf}" | cut -d"/" -f5)
+ CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1)
+ fi
+ echo $CTID $node
+ }""")
+
+ def prepare(self):
+ super(ProxmoxOVZ, self).prepare()
+ self.append(self.GET_PROXMOX_INFO)
+
+ def get_vzset_args(self, vps):
+ args = []
+ for resource, arg_name in self.RESOURCES:
+ try:
+ allocation = getattr(vps.resources, resource).allocated
+ except AttributeError:
+ pass
+ else:
+ args.append('--%s %i' % (arg_name, allocation))
+ return ' '.join(args)
+
+ def save(self, vps):
+ context = self.get_context(vps)
+ self.append('info=( $(get_vz_info %(hostname)s) )' % context)
+ vzset_args = self.get_vzset_args(vps)
+ if vzset_args:
+ context['vzset_args'] = vzset_args
+ self.append(textwrap.dedent("""\
+ cat << EOF | ssh root@${info[1]}
+ pvectl vzset ${info[0]} %(vzset_args)s
+ EOF""") % context
+ )
+ if hasattr(vps, 'password'):
+ context['password'] = vps.password.replace('$', '\\$')
+ self.append(textwrap.dedent("""\
+ cat << EOF | ssh root@${info[1]}
+ echo 'root:%(password)s' | vzctl exec ${info[0]} chpasswd -e
+ EOF""") % context
+ )
+ def get_context(self, vps):
+ return {
+ 'hostname': vps.hostname,
+ }
+
+
+# TODO rename to proxmox
class OpenVZTraffic(ServiceMonitor):
- """
- WARNING: Not fully implemeted
- """
model = 'vps.VPS'
resource = ServiceMonitor.TRAFFIC
monthly_sum_old_values = True
+ def prepare(self):
+ super(OpenVZTraffic, self).prepare()
+ self.append(ProxmoxOVZ.GET_PROXMOX_INFO)
+ self.append(textwrap.dedent("""
+ function monitor () {
+ object_id=$1
+ hostname=$2
+ info=( $(get_vz_info $hostname) )
+ cat << EOF | ssh root@${info[1]}
+ vzctl exec ${info[0]} cat /proc/net/dev \\
+ | grep venet0 \\
+ | tr ':' ' ' \\
+ | awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}'
+ EOF
+ }
+ """)
+ )
+
def process(self, line):
- """ diff with last stored value """
- object_id, value = line.split()
+ """ diff with last stored state """
+ object_id, value, state = super(OpenVZTraffic, self).process(line)
+ value = decimal.Decimal(value)
last = self.get_last_data(object_id)
- if not last or last.value > value:
- return object_id, value
- return object_id, value-last.value
+ if not last or last.state > value:
+ return object_id, value, value
+ return object_id, value-last.state, value
- def monitor(self, container):
- """ Get OpenVZ container traffic on a Proxmox +2.0 cluster """
- context = self.get_context(container)
- self.append(
- "CONF=$(grep -r 'HOSTNAME=\"%(hostname)s\"' /etc/pve/nodes/*/openvz/*.conf)" % context)
- self.append('NODE=$(echo "${CONF}" | cut -d"/" -f5)')
- self.append('CTID=$(echo "${CONF}" | cut -d"/" -f7 | cur -d"\." -f1)')
- self.append(
- "ssh root@${NODE} vzctl exec ${CTID} cat /proc/net/dev \\\n"
- " | grep venet0 \\\n"
- " | awk -F: '{print $2}' \\\n"
- " | awk '{print $1+$9}'")
+ def monitor(self, vps):
+ """ Get OpenVZ container traffic on a Proxmox cluster """
+ context = self.get_context(vps)
+ self.append('monitor %(object_id)s %(hostname)s' % context)
- def get_context(self, container):
- context = {
- 'hostname': container.hostname,
+ def get_context(self, vps):
+ return {
+ 'object_id': vps.id,
+ 'hostname': vps.hostname,
}
- return replace(context, "'", '"')
diff --git a/orchestra/contrib/vps/forms.py b/orchestra/contrib/vps/forms.py
deleted file mode 100644
index a6f12aab..00000000
--- a/orchestra/contrib/vps/forms.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from django import forms
-from django.contrib.auth.forms import ReadOnlyPasswordHashField
-from django.utils.translation import ugettext_lazy as _
-
-
-class VPSCreationForm(forms.ModelForm):
- password1 = forms.CharField(label=_("Password"),
- widget=forms.PasswordInput)
- password2 = forms.CharField(label=_("Password confirmation"),
- widget=forms.PasswordInput,
- help_text=_("Enter the same password as above, for verification."))
-
- class Meta:
- fields = ('username', 'account', 'type', 'template')
-
- def clean_password2(self):
- password1 = self.cleaned_data.get("password1")
- password2 = self.cleaned_data.get("password2")
- if password1 and password2 and password1 != password2:
- msg = _("The two password fields didn't match.")
- raise forms.ValidationError(msg)
- return password2
-
- def save(self, commit=True):
- vps = super(VPSCreationForm, self).save(commit=False)
- vps.set_password(self.cleaned_data["password1"])
- if commit:
- vps.save()
- return vps
-
-
-class VPSChangeForm(forms.ModelForm):
- password = ReadOnlyPasswordHashField(label=_("Password"),
- help_text=_("Raw passwords are not stored, so there is no way to see "
- "this user's password, but you can change the password "
- "using this form."))
-
- def clean_password(self):
- return self.initial["password"]
diff --git a/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py b/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py
new file mode 100644
index 00000000..c3569416
--- /dev/null
+++ b/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('vps', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='vps',
+ name='password',
+ ),
+ migrations.AlterField(
+ model_name='vps',
+ name='template',
+ field=models.CharField(default='debian7', verbose_name='template', choices=[('debian7', 'Debian 7 - Wheezy')], max_length=64, help_text='Initial template.'),
+ ),
+ ]
diff --git a/orchestra/contrib/vps/models.py b/orchestra/contrib/vps/models.py
index 27691f3c..93441fac 100644
--- a/orchestra/contrib/vps/models.py
+++ b/orchestra/contrib/vps/models.py
@@ -13,9 +13,8 @@ class VPS(models.Model):
type = models.CharField(_("type"), max_length=64, choices=settings.VPS_TYPES,
default=settings.VPS_DEFAULT_TYPE)
template = models.CharField(_("template"), max_length=64,
- choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE)
- password = models.CharField(_('password'), max_length=128,
- help_text=_("root password of this virtual machine"))
+ choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE,
+ help_text=_("Initial template."))
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
related_name='vpss')
diff --git a/orchestra/forms/options.py b/orchestra/forms/options.py
index e81e68da..afa0f6ec 100644
--- a/orchestra/forms/options.py
+++ b/orchestra/forms/options.py
@@ -60,7 +60,7 @@ class UserCreationForm(forms.ModelForm):
class UserChangeForm(forms.ModelForm):
password = auth_forms.ReadOnlyPasswordHashField(label=_("Password"),
help_text=_("Raw passwords are not stored, so there is no way to see "
- "this user's password, but you can change the password "
+ "this user's password, but you can change it by "
"using this form."))
def clean_password(self):
@@ -70,6 +70,13 @@ class UserChangeForm(forms.ModelForm):
return self.initial["password"]
+class NonStoredUserChangeForm(forms.ModelForm):
+ password = forms.CharField(label=_("Password"), required=False,
+ widget=SpanWidget(display='Unknown password'),
+ help_text=_("This service's password is not stored, so there is no way to see it, "
+ "but you can change it using this form."))
+
+
class ReadOnlyFormMixin(object):
"""
Mixin class for ModelForm or Form that provides support for SpanField on readonly fields
diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py
index ba631535..734ee474 100644
--- a/orchestra/utils/sys.py
+++ b/orchestra/utils/sys.py
@@ -142,7 +142,11 @@ def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', async
def sshrun(addr, command, *args, executable='bash', persist=False, **kwargs):
from .. import settings
- options = ['stricthostkeychecking=no']
+ options = [
+ 'stricthostkeychecking=no',
+ 'BatchMode=yes',
+ 'EscapeChar=none',
+ ]
if persist:
options.extend((
'ControlMaster=auto',