pedro
/
django-orchestra-test
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improved resource monitoring

This commit is contained in:
Marc 2014-07-09 16:17:43 +00:00
parent 1cd3092673
commit 9b9abc3c91
39 changed files with 420 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
TODO.md
View File

@ -47,3 +47,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* passlib; nano /usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py SortedDict -> collections.OrderedDict
* pip install pyinotify
* Backend.operations dynamically generated based on defined methods

3
orchestra/admin/decorators.py
View File

@ -8,12 +8,13 @@ from django.utils.encoding import force_text
def action_with_confirmation(action_name, extra_context={},
template='admin/controller/generic_confirmation.html'):
template='admin/orchestra/generic_confirmation.html'):
"""
Generic pattern for actions that needs confirmation step
If custom template is provided the form must contain:
<input type="hidden" name="post" value="generic_confirmation" />
"""
def decorator(func, extra_context=extra_context, template=template):
@wraps(func, assigned=available_attrs(func))
def inner(modeladmin, request, queryset):

40
orchestra/apps/databases/backends.py
View File

@ -1,18 +1,25 @@
from orchestra.apps.orchestration import ServiceBackend
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
class MySQLDBBackend(ServiceBackend):
class MySQLDBBackend(ServiceController):
verbose_name = "MySQL database"
model = 'databases.Database'
def save(self, database):
if database.type == database.MYSQL:
context = self.get_context(database)
self.append("mysql -e 'CREATE DATABASE `%(database)s`;'" % context)
self.append("mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
" TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context)
self.append(
"mysql -e 'CREATE DATABASE `%(database)s`;'" % context
)
self.append(
"mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
" TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
)
def delete(self, database):
if database.type == database.MYSQL:
@ -30,21 +37,27 @@ class MySQLDBBackend(ServiceBackend):
}
class MySQLUserBackend(ServiceBackend):
class MySQLUserBackend(ServiceController):
verbose_name = "MySQL user"
model = 'databases.DatabaseUser'
def save(self, database):
if database.type == database.MYSQL:
context = self.get_context(database)
self.append("mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context)
self.append("mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
" WHERE User=\"%(username)s\";'" % context)
self.append(
"mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context
)
self.append(
"mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
" WHERE User=\"%(username)s\";'" % context
)
def delete(self, database):
if database.type == database.MYSQL:
context = self.get_context(database)
self.append("mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context)
self.append(
"mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
)
def get_context(self, database):
return {
@ -54,7 +67,12 @@ class MySQLUserBackend(ServiceBackend):
}
class MySQLPermissionBackend(ServiceBackend):
class MySQLPermissionBackend(ServiceController):
model = 'databases.UserDatabaseRelation'
verbose_name = "MySQL permission"
class MysqlDisk(ServiceMonitor):
model = 'database.Database'
resource = ServiceMonitor.DISK
verbose_name = _("MySQL disk")

4
orchestra/apps/domains/backends.py
View File

@ -4,10 +4,10 @@ from django.utils.translation import ugettext_lazy as _
from . import settings
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
class Bind9MasterDomainBackend(ServiceBackend):
class Bind9MasterDomainBackend(ServiceController):
verbose_name = _("Bind9 master domain")
model = 'domains.Domain'
related_models = (

38
orchestra/apps/lists/backends.py
View File

@ -1,11 +1,41 @@
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
class MailmanBackend(ServiceBackend):
class MailmanBackend(ServiceController):
verbose_name = "Mailman"
model = 'lists.List'
def save(self, mailinglist):
pass
class MailmanTraffic(ServiceMonitor):
model = 'lists.List'
resource = ServiceMonitor.TRAFFIC
def process(self, output):
for line in output.readlines():
listname, value = line.strip().slpit()
def monitor(self, mailinglist):
self.append(
"LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
" | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
" | while read line; do \n"
" grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
" | sed 's/size=//' | sed 's/,//'\n"
"done)"
)
self.append(
'SUBS=""\n'
'while read LIST; do\n'
' NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
' SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
' if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
' SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
' fi\n'
' SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
' echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
'done <<< "$LISTS"'
)

2
orchestra/apps/orchestration/__init__.py
View File

@ -1 +1 @@
from .backends import ServiceBackend
from .backends import ServiceBackend, ServiceController

13
orchestra/apps/orchestration/admin.py
View File

@ -22,10 +22,11 @@ STATE_COLORS = {
class RouteAdmin(admin.ModelAdmin):
list_display = [
'id', 'backend', 'host', 'match', 'display_model', 'is_active'
'id', 'backend', 'host', 'match', 'display_model', 'display_actions',
'is_active'
]
list_editable = ['backend', 'host', 'match', 'is_active']
list_filter = ['backend', 'host', 'is_active']
list_filter = ['host', 'is_active', 'backend']
def display_model(self, route):
try:
@ -35,6 +36,14 @@ class RouteAdmin(admin.ModelAdmin):
display_model.short_description = _("model")
display_model.allow_tags = True
def display_actions(self, route):
try:
return '<br>'.join(route.get_backend().get_actions())
except KeyError:
return "<span style='color: red;'>NOT AVAILABLE</span>"
display_actions.short_description = _("actions")
display_actions.allow_tags = True
class BackendOperationInline(admin.TabularInline):
model = BackendOperation

16
orchestra/apps/orchestration/backends.py
View File

@ -23,6 +23,7 @@ class ServiceBackend(object):
function_method = methods.Python
type = 'task' # 'sync'
ignore_fields = []
actions = []
# TODO type: 'script', execution:'task'
@ -37,6 +38,10 @@ class ServiceBackend(object):
def __init__(self):
self.cmds = []
@classmethod
def get_actions(cls):
return [ action for action in cls.actions if action in dir(cls) ]
@classmethod
def get_name(cls):
return cls.__name__
@ -68,7 +73,7 @@ class ServiceBackend(object):
choices = []
for b in backends:
# don't evaluate b.verbose_name ugettext_lazy
verbose = getattr(b.verbose_name, '_proxy____args', [None])
verbose = getattr(b.verbose_name, '_proxy____args', [b.verbose_name])
if verbose[0]:
verbose = b.verbose_name
else:
@ -110,3 +115,12 @@ class ServiceBackend(object):
the service once in bulk operations
"""
pass
class ServiceController(ServiceBackend):
actions = ('save', 'delete')
@classmethod
def get_backends(cls):
""" filter controller classes """
return [ plugin for plugin in cls.plugins if ServiceController in plugin.__mro__ ]

29
orchestra/apps/orchestration/models.py
View File

@ -76,16 +76,14 @@ class BackendOperation(models.Model):
"""
Encapsulates an operation, storing its related object, the action and the backend.
"""
SAVE = 'save'
DELETE = 'delete'
ACTIONS = (
(SAVE, _("save")),
(DELETE, _("delete")),
)
SAVE = 'save'
MONITOR = 'monitor'
log = models.ForeignKey('orchestration.BackendLog', related_name='operations')
# TODO backend and backend_class() (like content_type)
backend_class = models.CharField(_("backend"), max_length=256)
action = models.CharField(_("action"), max_length=64, choices=ACTIONS)
action = models.CharField(_("action"), max_length=64)
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
instance = generic.GenericForeignKey('content_type', 'object_id')
@ -149,14 +147,21 @@ class Route(models.Model):
@classmethod
def get_servers(cls, operation):
backend_name = operation.backend.get_name()
# TODO use cached data sctructure and refactor
backend = operation.backend
servers = []
try:
routes = cls.objects.filter(is_active=True, backend=backend_name)
routes = cls.objects.filter(is_active=True, backend=backend.get_name())
except cls.DoesNotExist:
return []
safe_locals = { 'instance': operation.instance }
pks = [ route.pk for route in routes.all() if eval(route.match, safe_locals) ]
return [ route.host for route in routes.filter(pk__in=pks) ]
return servers
safe_locals = {
'instance': operation.instance
}
actions = backend.get_actions()
for route in routes:
if operation.action in actions and eval(route.match, safe_locals):
servers.append(route.host)
return servers
def get_backend(self):
for backend in ServiceBackend.get_backends():

3
orchestra/apps/resources/__init__.py
View File

@ -1 +1,4 @@
from .backends import ServiceMonitor
default_app_config = 'orchestra.apps.resources.apps.ResourcesConfig'

25
orchestra/apps/resources/admin.py
View File

@ -1,5 +1,3 @@
import sys
from django.contrib import admin
from django.contrib.contenttypes import generic
from django.utils.functional import cached_property
@ -7,9 +5,10 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.admin.filters import UsedContentTypeFilter
from orchestra.admin.utils import insertattr, get_modeladmin
from orchestra.utils import running_syncdb
from .forms import ResourceForm
from .models import Resource, ResourceAllocation, Monitor, MonitorData
from .models import Resource, ResourceData, MonitorData
class ResourceAdmin(admin.ModelAdmin):
@ -26,30 +25,24 @@ class ResourceAdmin(admin.ModelAdmin):
resources = obj.content_type.resource_set.filter(is_active=True)
inlines = []
for inline in modeladmin.inlines:
if inline.model is ResourceAllocation:
if inline.model is ResourceData:
inline = resource_inline_factory(resources)
inlines.append(inline)
modeladmin.inlines = inlines
class ResourceAllocationAdmin(admin.ModelAdmin):
list_display = ('id', 'resource', 'content_object', 'value')
class ResourceDataAdmin(admin.ModelAdmin):
list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object
list_filter = ('resource',)
class MonitorAdmin(admin.ModelAdmin):
list_display = ('backend', 'resource', 'crontab')
list_filter = ('backend', 'resource')
class MonitorDataAdmin(admin.ModelAdmin):
list_display = ('id', 'monitor', 'content_object', 'date', 'value')
list_display = ('id', 'monitor', 'date', 'value') # TODO content_object
list_filter = ('monitor',)
admin.site.register(Resource, ResourceAdmin)
admin.site.register(ResourceAllocation, ResourceAllocationAdmin)
admin.site.register(Monitor, MonitorAdmin)
admin.site.register(ResourceData, ResourceDataAdmin)
admin.site.register(MonitorData, MonitorDataAdmin)
@ -68,7 +61,7 @@ def resource_inline_factory(resources):
return forms
class ResourceInline(generic.GenericTabularInline):
model = ResourceAllocation
model = ResourceData
verbose_name_plural = _("resources")
form = ResourceForm
formset = ResourceInlineFormSet
@ -84,7 +77,7 @@ def resource_inline_factory(resources):
return ResourceInline
if not 'migrate' in sys.argv and not 'syncdb' in sys.argv:
if not running_syncdb():
# not run during syncdb
for resources in Resource.group_by_content_type():
inline = resource_inline_factory(resources)

11
orchestra/apps/resources/apps.py
View File

@ -1,6 +1,8 @@
from django.apps import AppConfig
from django.contrib.contenttypes import generic
from orchestra.utils import running_syncdb
class ResourcesConfig(AppConfig):
name = 'orchestra.apps.resources'
@ -9,7 +11,8 @@ class ResourcesConfig(AppConfig):
def ready(self):
from .models import Resource
# TODO execute on Resource.save()
relation = generic.GenericRelation('resources.ResourceAllocation')
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
model.add_to_class('allocations', relation)
if not running_syncdb():
relation = generic.GenericRelation('resources.ResourceData')
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
model.add_to_class('resources', relation)

27
orchestra/apps/resources/backends.py Normal file
View File

@ -0,0 +1,27 @@
from orchestra.apps.orchestration import ServiceBackend
class ServiceMonitor(ServiceBackend):
TRAFFIC = 'traffic'
DISK = 'disk'
MEMORY = 'memory'
CPU = 'cpu'
actions = ('monitor', 'resource_exceeded', 'resource_recovery')
@classmethod
def get_backends(cls):
""" filter monitor classes """
return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__]
def store(self, stdout):
""" object_id value """
for line in stdout.readlines():
line = line.strip()
object_id, value = line.split()
# TODO date
MonitorHistory.store(self.model, object_id, value, date)
def execute(self, server):
log = super(MonitorBackend, self).execute(server)
return log

22
orchestra/apps/resources/forms.py
View File

@ -7,23 +7,33 @@ from orchestra.forms.widgets import ShowTextWidget, ReadOnlyWidget
class ResourceForm(forms.ModelForm):
verbose_name = forms.CharField(label=_("Name"), widget=ShowTextWidget(bold=True),
required=False)
current = forms.CharField(label=_("Current"), widget=ShowTextWidget(),
used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(),
required=False)
value = forms.CharField(label=_("Allocation"))
last_update = forms.CharField(label=_("Last update"), widget=ShowTextWidget(),
required=False)
allocated = forms.IntegerField(label=_("Allocated"))
class Meta:
fields = ('verbose_name', 'current', 'value',)
fields = ('verbose_name', 'used', 'last_update', 'allocated',)
def __init__(self, *args, **kwargs):
self.resource = kwargs.pop('resource', None)
super(ResourceForm, self).__init__(*args, **kwargs)
if self.resource:
self.fields['verbose_name'].initial = self.resource.verbose_name
self.fields['current'].initial = self.resource.get_current()
self.fields['used'].initial = self.resource.get_current()
if self.resource.ondemand:
self.fields['value'].widget = ReadOnlyWidget('')
self.fields['allocated'].required = False
self.fields['allocated'].widget = ReadOnlyWidget(None, '')
else:
self.fields['value'].initial = self.resource.default_allocation
self.fields['allocated'].required = True
self.fields['allocated'].initial = self.resource.default_allocation
def has_changed(self):
""" Make sure resourcedata objects are created for all resources """
if not self.instance.pk:
return True
return super(ResourceForm, self).has_changed()
def save(self, *args, **kwargs):
self.instance.resource_id = self.resource.pk

96
orchestra/apps/resources/models.py
View File

@ -7,13 +7,25 @@ from django.core import validators
from django.utils.translation import ugettext_lazy as _
from djcelery.models import PeriodicTask, CrontabSchedule
from orchestra.models.fields import MultiSelectField
from orchestra.utils.apps import autodiscover
from .backends import ServiceMonitor
class Resource(models.Model):
MONTHLY = 'MONTHLY'
"""
Defines a resource, a resource is basically an interpretation of data
gathered by a Monitor
"""
LAST = 'LAST'
MONTHLY_SUM = 'MONTHLY_SUM'
MONTHLY_AVG = 'MONTHLY_AVG'
PERIODS = (
(MONTHLY, _('Monthly')),
(LAST, _("Last")),
(MONTHLY_SUM, _("Monthly Sum")),
(MONTHLY_AVG, _("Monthly Average")),
)
name = models.CharField(_("name"), max_length=32, unique=True,
@ -24,11 +36,14 @@ class Resource(models.Model):
verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True)
content_type = models.ForeignKey(ContentType) # TODO filter by servicE?
period = models.CharField(_("period"), max_length=16, choices=PERIODS,
default=MONTHLY)
ondemand = models.BooleanField(default=False)
default_allocation = models.PositiveIntegerField(null=True, blank=True)
is_active = models.BooleanField(default=True)
disable_trigger = models.BooleanField(default=False)
default=LAST)
ondemand = models.BooleanField(_("on demand"), default=False)
default_allocation = models.PositiveIntegerField(_("default allocation"),
null=True, blank=True)
is_active = models.BooleanField(_("is active"), default=True)
disable_trigger = models.BooleanField(_("disable trigger"), default=False)
monitors = MultiSelectField(_("monitors"), max_length=256,
choices=ServiceMonitor.get_choices())
def __unicode__(self):
return self.name
@ -53,47 +68,58 @@ class Resource(models.Model):
today = datetime.date.today()
result = 0
has_result = False
for monitor in self.monitors.all():
has_result = True
if self.period == self.MONTHLY:
data = monitor.dataset.filter(date__year=today.year,
date__month=today.month)
result += data.aggregate(models.Sum('value'))['value__sum']
for monitor in self.monitors:
dataset = MonitorData.objects.filter(monitor=monitor)
if self.period == self.MONTHLY_AVG:
try:
last = dataset.latest()
except MonitorData.DoesNotExist:
continue
has_result = True
epoch = datetime(year=today.year, month=today.month, day=1)
total = (epoch-last.date).total_seconds()
dataset = dataset.filter(date__year=today.year,
date__month=today.month)
for data in dataset:
slot = (previous-data.date).total_seconds()
result += data.value * slot/total
elif self.period == self.MONTHLY_SUM:
data = dataset.filter(date__year=today.year,
date__month=today.month)
value = data.aggregate(models.Sum('value'))['value__sum']
if value:
has_result = True
result += value
elif self.period == self.LAST:
try:
result += dataset.latest().value
except MonitorData.DoesNotExist:
continue
has_result = True
else:
raise NotImplementedError("%s support not implemented" % self.period)
return result if has_result else None
class ResourceAllocation(models.Model):
class ResourceData(models.Model):
""" Stores computed resource usage and allocation """
resource = models.ForeignKey(Resource)
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
value = models.PositiveIntegerField()
used = models.PositiveIntegerField(null=True)
last_update = models.DateTimeField(null=True)
allocated = models.PositiveIntegerField(null=True)
content_object = generic.GenericForeignKey()
class Meta:
unique_together = ('resource', 'content_type', 'object_id')
autodiscover('monitors')
class Monitor(models.Model):
backend = models.CharField(_("backend"), max_length=256,)
# choices=MonitorBackend.get_choices())
resource = models.ForeignKey(Resource, related_name='monitors')
crontab = models.ForeignKey(CrontabSchedule)
class Meta:
unique_together=('backend', 'resource')
def __unicode__(self):
return self.backend
verbose_name_plural = _("resource data")
class MonitorData(models.Model):
monitor = models.ForeignKey(Monitor, related_name='dataset')
""" Stores monitored data """
monitor = models.CharField(_("monitor"), max_length=256,
choices=ServiceMonitor.get_choices())
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
date = models.DateTimeField(auto_now_add=True)
@ -101,5 +127,9 @@ class MonitorData(models.Model):
content_object = generic.GenericForeignKey()
class Meta:
get_latest_by = 'date'
verbose_name_plural = _("monitor data")
def __unicode__(self):
return str(self.monitor)

21
orchestra/apps/resources/serializers.py
View File

@ -1,27 +1,24 @@
from rest_framework import serializers
from orchestra.api import router
from orchestra.utils import running_syncdb
from .models import Resource, ResourceAllocation
from .models import Resource, ResourceData
class ResourceSerializer(serializers.ModelSerializer):
name = serializers.SerializerMethodField('get_name')
current = serializers.SerializerMethodField('get_current')
allocation = serializers.IntegerField(source='value')
class Meta:
model = ResourceAllocation
fields = ('name', 'current', 'allocation')
model = ResourceData
fields = ('name', 'used', 'allocated')
read_only_fields = ('used',)
def get_name(self, instance):
return instance.resource.name
def get_current(self, instance):
return instance.resource.get_current()
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
router.insert(model, 'resources', ResourceSerializer, required=False,
source='allocations')
if not running_syncdb():
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
router.insert(model, 'resources', ResourceSerializer, required=False)

14
orchestra/apps/resources/tasks.py Normal file
View File

@ -0,0 +1,14 @@
from celery import shared_task
from .backends import ServiceMonitor
@shared_task
def monitor(backend_name):
routes = Route.objects.filter(is_active=True, backend=backend_name)
for route in routes:
pass
for backend in ServiceMonitor.get_backends():
if backend.get_name() == backend_name:
# TODO execute monitor BackendOperation
pass

14
orchestra/apps/users/admin.py
View File

@ -15,7 +15,7 @@ from .roles.filters import role_list_filter_factory
class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
list_display = ('username', 'is_main')
list_display = ('username', 'display_is_main')
list_filter = ('is_staff', 'is_superuser', 'is_active')
fieldsets = (
(None, {
@ -25,7 +25,7 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
'fields': ('first_name', 'last_name', 'email')
}),
(_("Permissions"), {
'fields': ('is_active', 'is_staff', 'is_superuser', 'is_admin', 'is_main')
'fields': ('is_active', 'is_staff', 'is_superuser', 'display_is_main')
}),
(_("Important dates"), {
'fields': ('last_login', 'date_joined')
@ -38,7 +38,7 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
}),
)
search_fields = ['username', 'account__user__username']
readonly_fields = ('is_main', 'account_link')
readonly_fields = ('display_is_main', 'account_link')
change_readonly_fields = ('username',)
filter_horizontal = ()
add_form = UserCreationForm
@ -46,10 +46,10 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
roles = []
ordering = ('-id',)
def is_main(self, user):
return user.account.user == user
is_main.boolean = True
def display_is_main(self, instance):
return instance.is_main
display_is_main.short_description = _("is main")
display_is_main.boolean = True
def get_urls(self):
""" Returns the additional urls for the change view links """

24
orchestra/apps/users/backends.py
View File

@ -1,11 +1,12 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
class SystemUserBackend(ServiceBackend):
class SystemUserBackend(ServiceController):
verbose_name = _("System User")
model = 'users.User'
ignore_fields = ['last_login']
@ -39,3 +40,22 @@ class SystemUserBackend(ServiceBackend):
}
context['home'] = settings.USERS_SYSTEMUSER_HOME % context
return context
class SystemUserDisk(ServiceMonitor):
model = 'users.User'
resource = ServiceMonitor.DISK
verbose_name = _('System user disk')
def monitor(self, user):
context = self.get_context(user)
self.append("du -s %(home)s | {\n"
" read value\n"
" echo '%(username)s' $value\n"
"}" % context)
def process(self, output):
# TODO transaction
for line in output.readlines():
username, value = line.strip().slpit()
History.store(object_id=user_id, value=value)

4
orchestra/apps/users/models.py
View File

@ -36,6 +36,10 @@ class User(auth.AbstractBaseUser):
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = ['email']
@property
def is_main(self):
return self.account.user == self
def get_full_name(self):
full_name = '%s %s' % (self.first_name, self.last_name)
return full_name.strip() or self.username

17
orchestra/apps/users/roles/mail/backends.py
View File

@ -3,12 +3,13 @@ import os
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
class MailSystemUserBackend(ServiceBackend):
class MailSystemUserBackend(ServiceController):
verbose_name = _("Mail system user")
model = 'mail.Mailbox'
@ -62,7 +63,7 @@ class MailSystemUserBackend(ServiceBackend):
return context
class PostfixAddressBackend(ServiceBackend):
class PostfixAddressBackend(ServiceController):
verbose_name = _("Postfix address")
model = 'mail.Address'
@ -132,12 +133,12 @@ class PostfixAddressBackend(ServiceBackend):
return context
class AutoresponseBackend(ServiceBackend):
class AutoresponseBackend(ServiceController):
verbose_name = _("Mail autoresponse")
model = 'mail.Autoresponse'
def save(self, autoresponse):
pass
def delete(self, autoresponse):
pass
class MailDisk(ServiceMonitor):
model = 'email.Mailbox'
resource = ServiceMonitor.DISK
verbose_name = _("Mail disk")

20
orchestra/apps/vps/backends.py Normal file
View File

@ -0,0 +1,20 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.resources import ServiceMonitor
class OpenVZDisk(ServiceMonitor):
model = 'vps.VPS'
resource = ServiceMonitor.DISK
class OpenVZMemory(ServiceMonitor):
model = 'vps.VPS'
resource = ServiceMonitor.MEMORY
class OpenVZTraffic(ServiceMonitor):
model = 'vps.VPS'
resource = ServiceMonitor.TRAFFIC

7
orchestra/apps/webapps/backends/awstats.py
View File

@ -1,12 +1,9 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
class AwstatsBackend(WebAppServiceMixin, ServiceBackend):
class AwstatsBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("Awstats")
def save(self, webapp):
pass

5
orchestra/apps/webapps/backends/dokuwikimu.py
View File

@ -1,11 +1,12 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
class DokuWikiMuBackend(WebAppServiceMixin, ServiceBackend):
class DokuWikiMuBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("DokuWiki multisite")
def save(self, webapp):

4
orchestra/apps/webapps/backends/drupalmu.py
View File

@ -2,13 +2,13 @@ import os
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
class DrupalMuBackend(WebAppServiceMixin, ServiceBackend):
class DrupalMuBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("Drupal multisite")
def save(self, webapp):

4
orchestra/apps/webapps/backends/phpfcgid.py
View File

@ -2,13 +2,13 @@ import os
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
class PHPFcgidBackend(WebAppServiceMixin, ServiceBackend):
class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("PHP-Fcgid")
def save(self, webapp):

4
orchestra/apps/webapps/backends/phpfpm.py
View File

@ -3,13 +3,13 @@ import os
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
class PHPFPMBackend(WebAppServiceMixin, ServiceBackend):
class PHPFPMBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("PHP-FPM")
def save(self, webapp):

4
orchestra/apps/webapps/backends/static.py
View File

@ -1,11 +1,11 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
class StaticBackend(WebAppServiceMixin, ServiceBackend):
class StaticBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("Static")
def save(self, webapp):

4
orchestra/apps/webapps/backends/wordpressmu.py
View File

@ -4,13 +4,13 @@ import sys
import requests
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
class WordpressMuBackend(WebAppServiceMixin, ServiceBackend):
class WordpressMuBackend(WebAppServiceMixin, ServiceController):
verbose_name = _("Wordpress multisite")
@property

60
orchestra/apps/websites/backends/apache.py
View File

@ -3,12 +3,13 @@ import os
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from .. import settings
class Apache2Backend(ServiceBackend):
class Apache2Backend(ServiceController):
model = 'websites.Website'
related_models = (('websites.Content', 'website'),)
verbose_name = _("Apache 2")
@ -173,3 +174,58 @@ class Apache2Backend(ServiceBackend):
'fpm_port': content.webapp.get_fpm_port(),
})
return context
class Apache2Traffic(ServiceMonitor):
model = 'websites.Website'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Apache 2 Traffic")
def monitor(self, site):
context = self.get_context(site)
self.append("""
awk 'BEGIN {
ini = "%(start_date)s";
end = "%(end_date)s";
months["Jan"]="01";
months["Feb"]="02";
months["Mar"]="03";
months["Apr"]="04";
months["May"]="05";
months["Jun"]="06";
months["Jul"]="07";
months["Aug"]="08";
months["Sep"]="09";
months["Oct"]="10";
months["Nov"]="11";
months["Dec"]="12";
} {
date = substr($4,2)
year = substr(date,8,4)
month = months[substr(date,4,3)];
day = substr(date,1,2)
hour = substr(date,13,2)
minute = substr(date,16,2)
second = substr(date,19,2);
line_date = year month day hour minute second
if ( line_date > ini && line_date < end)
if ( $10 == "" )
sum+=$9
else
sum+=$10;
} END {
print sum;
}' %(log_file)s | {
read value
echo %(site_name)s $value
}
""" % context)
def get_context(self, site):
return {
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
'start_date': '',
'end_date': '',
'site_name': '',
}

4
orchestra/apps/websites/backends/webalizer.py
View File

@ -3,12 +3,12 @@ from functools import partial
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceBackend
from orchestra.apps.orchestration import ServiceController
from .. import settings
class WebalizerBackend(ServiceBackend):
class WebalizerBackend(ServiceController):
verbose_name = _("Webalizer")
model = 'websites.Content'

291
orchestra/apps/websites/resources.py
View File

@ -1,291 +0,0 @@
from . import settings
class ServiceBackend(object):
"""
Service management backend base class
It uses the _unit of work_ design principle, which allows bulk operations to
be conviniently supported. Each backend generates the configuration for all
the changes of all modified objects, reloading the daemon just once.
"""
verbose_name = None
model = None
related_models = () # ((model, accessor__attribute),)
script_method = methods.BashSSH
function_method = methods.Python
type = 'task' # 'sync'
ignore_fields = []
# TODO type: 'script', execution:'task'
__metaclass__ = plugins.PluginMount
def __unicode__(self):
return type(self).__name__
def __str__(self):
return unicode(self)
def __init__(self):
self.cmds = []
@classmethod
def get_name(cls):
return cls.__name__
@classmethod
def is_main(cls, obj):
opts = obj._meta
return cls.model == '%s.%s' % (opts.app_label, opts.object_name)
@classmethod
def get_related(cls, obj):
opts = obj._meta
model = '%s.%s' % (opts.app_label, opts.object_name)
for rel_model, field in cls.related_models:
if rel_model == model:
related = obj
for attribute in field.split('__'):
related = getattr(related, attribute)
return related
return None
@classmethod
def get_backends(cls):
return cls.plugins
@classmethod
def get_choices(cls):
backends = cls.get_backends()
choices = ( (b.get_name(), b.verbose_name or b.get_name()) for b in backends )
return sorted(choices, key=lambda e: e[1])
def get_banner(self):
time = datetime.now().strftime("%h %d, %Y %I:%M:%S")
return "Generated by Orchestra %s" % time
def append(self, *cmd):
# aggregate commands acording to its execution method
if isinstance(cmd[0], basestring):
method = self.script_method
cmd = cmd[0]
else:
method = self.function_method
cmd = partial(*cmd)
if not self.cmds or self.cmds[-1][0] != method:
self.cmds.append((method, [cmd]))
else:
self.cmds[-1][1].append(cmd)
def execute(self, server):
from .models import BackendLog
state = BackendLog.STARTED if self.cmds else BackendLog.SUCCESS
log = BackendLog.objects.create(backend=self.get_name(), state=state, server=server)
for method, cmds in self.cmds:
method(log, server, cmds)
if log.state != BackendLog.SUCCESS:
break
return log
def ServiceController(ServiceBackend):
def save(self, obj)
raise NotImplementedError
def delete(self, obj):
raise NotImplementedError
def commit(self):
"""
apply the configuration, usually reloading a service
reloading a service is done in a separated method in order to reload
the service once in bulk operations
"""
pass
class ServiceMonitor(ServiceBackend):
TRAFFIC = 'traffic'
DISK = 'disk'
MEMORY = 'memory'
CPU = 'cpu'
def prepare(self):
pass
def store(self, stdout):
""" object_id value """
for line in stdout.readlines():
line = line.strip()
object_id, value = line.split()
# TODO date
MonitorHistory.store(self.model, object_id, value, date)
def monitor(self, obj):
raise NotImplementedError
def trigger(self, obj):
raise NotImplementedError
def execute(self, server):
log = super(MonitorBackend, self).execute(server)
return log
class AccountDisk(MonitorBackend):
model = 'accounts.Account'
resource = MonitorBackend.DISK
verbose_name = 'Disk'
def monitor(self, user):
context = self.get_context(user)
self.append("du -s %(home)s | {\n"
" read value\n"
" echo '%(username)s' $value\n"
"}" % context)
def process(self, output):
# TODO transaction
for line in output.readlines():
username, value = line.strip().slpit()
History.store(object_id=user_id, value=value)
class MailmanTraffic(MonitorBackend):
model = 'lists.List'
resource = MonitorBackend.TRAFFIC
def process(self, output):
for line in output.readlines():
listname, value = line.strip().slpit()
def monitor(self, mailinglist):
self.append("LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
" | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
" | while read line; do \n"
" grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
" | sed 's/size=//' | sed 's/,//'\n"
"done)")
self.append('SUBS=""\n'
'while read LIST; do\n'
' NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
' SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
' if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
' SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
' fi\n'
' SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
' echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
'done <<< "$LISTS"')
class MailDisk(MonitorBackend):
model = 'email.Mailbox'
resource = MonitorBackend.DISK
verbose_name = _("Mail disk")
def process(self, output):
pass
def monitor(self, mail):
pass
class MysqlDisk(MonitorBackend):
model = 'database.Database'
resource = MonitorBackend.DISK
verbose_name = _("MySQL disk")
def process(self, output):
pass
def monitor(self, db):
pass
class OpenVZDisk(MonitorBackend):
model = 'vps.VPS'
resource = MonitorBackend.DISK
class OpenVZMemory(MonitorBackend):
model = 'vps.VPS'
resource = MonitorBackend.MEMORY
class OpenVZTraffic(MonitorBackend):
model = 'vps.VPS'
resource = MonitorBackend.TRAFFIC
class Apache2Traffic(MonitorBackend):
model = 'websites.Website'
resource = MonitorBackend.TRAFFIC
verbose_name = _("Apache2 Traffic")
def monitor(self, site):
context = self.get_context(site)
self.append("""
awk 'BEGIN {
ini = "%(start_date)s";
end = "%(end_date)s";
months["Jan"]="01";
months["Feb"]="02";
months["Mar"]="03";
months["Apr"]="04";
months["May"]="05";
months["Jun"]="06";
months["Jul"]="07";
months["Aug"]="08";
months["Sep"]="09";
months["Oct"]="10";
months["Nov"]="11";
months["Dec"]="12";
} {
date = substr($4,2)
year = substr(date,8,4)
month = months[substr(date,4,3)];
day = substr(date,1,2)
hour = substr(date,13,2)
minute = substr(date,16,2)
second = substr(date,19,2);
line_date = year month day hour minute second
if ( line_date > ini && line_date < end)
if ( $10 == "" )
sum+=$9
else
sum+=$10;
} END {
print sum;
}' %(log_file)s | {
read value
echo %(site_name)s $value
}
""" % context)
def trigger(self, site):
pass
def get_context(self, site):
return {
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name)
}
# start_date and end_date expected format: YYYYMMDDhhmmss
function get_traffic(){
RESULT=$(get_traffic)
if [[ $RESULT ]]; then
echo $RESULT
else
echo 0
fi
return 0

10
orchestra/forms/widgets.py
View File

@ -4,6 +4,11 @@ from django.utils.encoding import force_text
class ShowTextWidget(forms.Widget):
def __init__(self, *args, **kwargs):
for kwarg in ['bold', 'warning', 'hidden']:
setattr(self, kwarg, kwargs.pop(kwarg, False))
super(ShowTextWidget, self).__init__(*args, **kwargs)
def render(self, name, value, attrs):
value = force_text(value)
if value is None:
@ -20,11 +25,6 @@ class ShowTextWidget(forms.Widget):
final_value = u'%s<input type="hidden" name="%s" value="%s"/>' % (final_value, name, value)
return mark_safe(final_value)
def __init__(self, *args, **kwargs):
for kwarg in ['bold', 'warning', 'hidden']:
setattr(self, kwarg, kwargs.pop(kwarg, False))
super(ShowTextWidget, self).__init__(*args, **kwargs)
def _has_changed(self, initial, data):
return False

2
orchestra/models/fields.py
View File

@ -54,4 +54,4 @@ class MultiSelectField(models.CharField):
if isinstalled('south'):
from south.modelsinspector import add_introspection_rules
add_introspection_rules([], ["^controller\.models\.fields\.MultiSelectField"])
add_introspection_rules([], ["^orchestra\.models\.fields\.MultiSelectField"])

10
orchestra/static/orchestra/css/adminextraprettystyle.css
View File

@ -5,18 +5,18 @@ body {
#header #branding h1 {
margin: 0;
padding: 5px 10px;
background: transparent url(/static/orchestra/images/orchestra-logo.png) 10px 5px no-repeat;
padding: 2px 10px;
background: transparent url(/static/orchestra/images/orchestra-logo.png) 10px 2px no-repeat;
text-indent: 0;
height: 31px;
font-size: 18px;
font-weight: bold;
font-size: 16px;
/* font-weight: bold;*/
padding-left: 50px;
line-height: 30px;
}
#branding h1, #branding h1 a:link, #branding h1 a:visited {
color: #707070;
color: #555;
}

BIN
orchestra/static/orchestra/images/orchestra-logo.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

4
orchestra/static/orchestra/images/orchestra-logo.svg
View File

@ -16,8 +16,8 @@
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="orchestra-logo.svg"
inkscape:export-filename="/home/glic3rinu/orchestra/django-orchestra/orchestra/static/orchestra/images/orchestra-logo.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
inkscape:export-xdpi="81.290321"
inkscape:export-ydpi="81.290321">
<defs
id="defs4">
<linearGradient

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

2
orchestra/templatetags/utils.py
View File

@ -9,7 +9,7 @@ register = template.Library()
@register.simple_tag(name="version")
def controller_version():
def orchestra_version():
return get_version()

4
orchestra/utils/options.py
View File

@ -1,3 +1,4 @@
import sys
import urlparse
from django.core.mail import EmailMultiAlternatives
@ -37,3 +38,6 @@ def send_email_template(template, context, to, email_from=None, html=None):
msg.attach_alternative(html_message, "text/html")
msg.send()
def running_syncdb():
return 'migrate' in sys.argv or 'syncdb' in sys.argv