pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

More robust bash backends using heredoc

Browse source
This commit is contained in:
Marc Aymerich 2015-05-21 13:34:12 +00:00
parent 2ac063ef23
commit eec571d56f
12 changed files with 71 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
TODO.md
View file

@ -284,7 +284,6 @@ https://code.djangoproject.com/ticket/24576
# TODO orchestra related services code reload: celery/uwsgi reloading find aonther way without root and implement reload
# password validation cracklib on change password form=?????
# reset setting button
# admin edit relevant djanog settings
@ -301,8 +300,7 @@ https://code.djangoproject.com/ticket/24576
# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
# Ship orchestra production-ready (no DEBUG etc)
# import module and sed
# if setting.value == default. remove
# Settings.parser.changes: if setting.value == default. remove
# reload generic admin view ?redirect=http...
# inspecting django db connection for asserting db readines? or performing a query
# wake up django mailer on send_mail
@ -363,7 +361,6 @@ pip3 install https://github.com/fantix/gevent/archive/master.zip
# SIgnal handler for notify workers to reload stuff, like resource sync: https://docs.python.org/2/library/signal.html
# BUG Delete related services also deletes account!
# auto apend trailing slash
# get_related service__rates__isnull=TRue is that correct?
@ -378,3 +375,7 @@ method(
# dovecot sieve only allolws one fucking active script. refactor mailbox shit to replace active script symlink by orchestra. Create a generic wrapper that includes al filters (rc, imp and orchestra)
http://wiki2.dovecot.org/Pigeonhole/Sieve/Examples
# orders ignorign default filter is not very effective, because of selecting all orders for billing will select ignored too

7
orchestra/contrib/accounts/admin.py
View file

@ -8,6 +8,7 @@ from django.contrib import admin, messages
from django.contrib.admin.utils import unquote
from django.contrib.auth import admin as auth
from django.http import HttpResponseRedirect
from django.templatetags.static import static
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
@ -160,11 +161,11 @@ class AccountAdminMixin(object):
def display_active(self, instance):
if not instance.is_active:
return '<img src="/static/admin/img/icon-no.gif" alt="False">'
return '<img src="%s" alt="False">' % static('admin/img/icon-no.gif')
elif not instance.account.is_active:
msg = _("Account disabled")
return '<img src="/static/admin/img/icon-unknown.gif" alt="False" title="%s">' % msg
return '<img src="/static/admin/img/icon-yes.gif" alt="True">'
return '<img src="%s" alt="False" title="%s">' % (static('admin/img/icon-unknown.gif'), msg)
return '<img src="%s" alt="False">' % static('admin/img/icon-yes.gif')
display_active.short_description = _("active")
display_active.allow_tags = True
display_active.admin_order_field = 'is_active'

24
orchestra/contrib/domains/backends.py
View file

@ -4,7 +4,7 @@ import textwrap
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.orchestration import Operation
from orchestra.utils.python import OrderedSet
@ -44,9 +44,11 @@ class Bind9MasterDomainBackend(ServiceController):
def update_zone(self, domain, context):
context['zone'] = ';; %(banner)s\n' % context
context['zone'] += domain.render_zone().replace("'", '"')
self.append(textwrap.dedent("""\
echo -e '%(zone)s' > %(zone_path)s.tmp
context['zone'] += domain.render_zone()
self.append(textwrap.dedent("""
cat << 'EOF' > %(zone_path)s.tmp
%(zone)s
EOF
diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1
# Because bind reload will not display any fucking error
named-checkzone -k fail -n fail %(name)s %(zone_path)s.tmp
@ -55,8 +57,10 @@ class Bind9MasterDomainBackend(ServiceController):
)
def update_conf(self, context):
self.append(textwrap.dedent("""\
conf='%(conf)s'
self.append(textwrap.dedent("""
read -r -d '' conf << 'EOF' || true
%(conf)s
EOF
sed '/zone "%(name)s".*/,/^\s*};\s*$/!d' %(conf_path)s | diff -B -I"^\s*//" - <(echo "${conf}") || {
sed -i -e '/zone\s\s*"%(name)s".*/,/^\s*};/d' \\
-e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s
@ -82,7 +86,7 @@ class Bind9MasterDomainBackend(ServiceController):
if context['name'][0] in ('*', '_'):
# These can never be top level domains
return
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
sed -e '/zone\s\s*"%(name)s".*/,/^\s*};\s*$/d' \\
-e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s > %(conf_path)s.tmp""") % context
)
@ -141,7 +145,7 @@ class Bind9MasterDomainBackend(ServiceController):
'also_notify': '; '.join(slaves) + ';' if slaves else '',
'conf_path': self.CONF_PATH,
}
context['conf'] = textwrap.dedent("""
context['conf'] = textwrap.dedent("""\
zone "%(name)s" {
// %(banner)s
type master;
@ -150,7 +154,7 @@ class Bind9MasterDomainBackend(ServiceController):
also-notify { %(also_notify)s };
notify yes;
};""") % context
return replace(context, "'", '"')
return context
class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
@ -200,4 +204,4 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
masters { %(masters)s; };
allow-notify { %(masters)s; };
};""") % context
return replace(context, "'", '"')
return context

2
orchestra/contrib/domains/models.py
View file

@ -107,7 +107,7 @@ class Domain(models.Model):
zone += subdomain.render_records()
for subdomain in sorted(tail, key=lambda x: len(x.name), reverse=True):
zone += subdomain.render_records()
return zone
return zone.strip()
def refresh_serial(self):
""" Increases the domain serial number by one """

3
orchestra/contrib/issues/forms.py
View file

@ -3,6 +3,7 @@ from django.contrib.auth import get_user_model
from django.utils.html import strip_tags
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from django.templatetags.static import static
from markdown import markdown
from orchestra.forms.widgets import SpanWidget
@ -13,7 +14,7 @@ from .models import Queue, Ticket
class MarkDownWidget(forms.Textarea):
""" MarkDown textarea widget with syntax preview """
markdown_url = '/static/issues/markdown_syntax.html'
markdown_url = static('issues/markdown_syntax.html')
markdown_help_text = (
'<a href="%s" onclick=\'window.open("%s", "", "resizable=yes, '
'location=no, width=300, height=640, menubar=no, status=no, scrollbars=yes"); '

30
orchestra/contrib/mailboxes/backends.py
View file

@ -6,7 +6,7 @@ import textwrap
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from . import settings
@ -22,7 +22,7 @@ class SieveFilteringMixin(object):
for box in re.findall(r'fileinto\s+"([^"]+)"', content):
# create mailboxes if fileinfo is provided witout ':create' option
context['box'] = box
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
mkdir -p %(maildir)s/.%(box)s
chown %(user)s:%(group)s %(maildir)s/.%(box)s
if [[ ! $(grep '%(box)s' %(maildir)s/subscriptions) ]]; then
@ -34,9 +34,11 @@ class SieveFilteringMixin(object):
context['filtering_cpath'] = re.sub(r'\.sieve$', '.svbin', context['filtering_path'])
if content:
context['filtering'] = ('# %(banner)s\n' + content) % context
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
mkdir -p $(dirname '%(filtering_path)s')
echo '%(filtering)s' > %(filtering_path)s
cat << 'EOF' > %(filtering_path)s
%(filtering)s
EOF
sievec %(filtering_path)s
chown %(user)s:%(group)s {%(filtering_path)s,%(filtering_cpath)s}
""") % context
@ -66,7 +68,9 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
# Fucking postfix SASL caches credentials
old_password=$(grep "^%(user)s:" /etc/shadow|cut -d':' -f2)
usermod %(user)s --password '%(password)s' --shell %(initial_shell)s
[[ "$old_password" != "%(password)s" ]] && RESTART_POSTFIX=1
if [[ "$old_password" != "%(password)s" ]]; then
RESTART_POSTFIX=1
fi
else
useradd %(user)s --home %(home)s --password '%(password)s'
fi
@ -118,7 +122,7 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
'initial_shell': self.SHELL,
'banner': self.get_banner(),
}
return replace(context, "'", '"')
return context
class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceController):
@ -158,7 +162,7 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl
def delete(self, mailbox):
context = self.get_context(mailbox)
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
nohup bash -c 'sleep 2 && killall -u %(uid)s -s KILL' &> /dev/null &
killall -u %(uid)s || true
sed -i '/^%(user)s:.*/d' %(passwd_path)s
@ -186,7 +190,7 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl
context = {
'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH
}
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && {
postmap %(virtual_mailbox_maps)s
}""") % context
@ -212,7 +216,7 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl
'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context),
'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context,
})
return replace(context, "'", '"')
return context
class PostfixAddressVirtualDomainBackend(ServiceController):
@ -248,7 +252,7 @@ class PostfixAddressVirtualDomainBackend(ServiceController):
def exclude_virtual_alias_domain(self, context):
domain = context['domain']
if self.is_last_domain(domain):
self.append(textwrap.dedent("""\
self.append(textwrap.dedent("""
if [[ $(grep '^%(domain)s\s*$' %(virtual_alias_domains)s) ]]; then
sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
@ -288,7 +292,7 @@ class PostfixAddressVirtualDomainBackend(ServiceController):
'email': address.email,
'local_domain': settings.MAILBOXES_LOCAL_DOMAIN,
})
return replace(context, "'", '"')
return context
class PostfixAddressBackend(PostfixAddressVirtualDomainBackend):
@ -401,7 +405,7 @@ class DovecotMaildirDisk(ServiceMonitor):
'object_id': mailbox.pk
}
context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context
return replace(context, "'", '"')
return context
class PostfixMailscannerTraffic(ServiceMonitor):
@ -552,4 +556,4 @@ class PostfixMailscannerTraffic(ServiceMonitor):
'object_id': mailbox.pk,
'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
return replace(context, "'", '"')
return context

2
orchestra/contrib/systemusers/backends.py
View file

@ -284,7 +284,7 @@ class Exim4Traffic(ServiceMonitor):
'object_id': user.pk,
'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
return replace(context, "'", '"')
return context
class VsFTPdTraffic(ServiceMonitor):

4
orchestra/contrib/webapps/backends/__init__.py
View file

@ -1,8 +1,6 @@
import pkgutil
import textwrap
from orchestra.contrib.orchestration.backends import replace
from .. import settings
@ -57,7 +55,7 @@ class WebAppServiceMixin(object):
'is_mounted': webapp.content_set.exists(),
}
context['deleted_app_path'] = settings.WEBAPPS_MOVE_ON_DELETE_PATH % context
return replace(context, "'", '"')
return context
for __, module_name, __ in pkgutil.walk_packages(__path__):

21
orchestra/contrib/webapps/backends/php.py
View file

@ -5,7 +5,7 @@ from collections import OrderedDict
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.orchestration import ServiceController
from . import WebAppServiceMixin
from .. import settings
@ -43,8 +43,10 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
self.set_under_construction(context)
def save_fpm(self, webapp, context):
self.append(textwrap.dedent("""\
fpm_config='%(fpm_config)s'
self.append(textwrap.dedent("""
read -r -d '' fpm_config << 'EOF' || true
%(fpm_config)s
EOF
{
echo -e "${fpm_config}" | diff -N -I'^\s*;;' %(fpm_path)s -
} || {
@ -57,7 +59,9 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
def save_fcgid(self, webapp, context):
self.append("mkdir -p %(wrapper_dir)s" % context)
self.append(textwrap.dedent("""\
wrapper='%(wrapper)s'
read -r -d '' wrapper << 'EOF' || true
%(wrapper)s
EOF
{
echo -e "${wrapper}" | diff -N -I'^\s*#' %(wrapper_path)s -
} || {
@ -73,8 +77,10 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
self.append("chmod 550 %(wrapper_path)s" % context)
self.append("chown -R %(user)s:%(group)s %(wrapper_dir)s" % context)
if context['cmd_options']:
self.append(textwrap.dedent("""
cmd_options='%(cmd_options)s'
self.append(textwrap.dedent("""\
read -r -d '' cmd_options << 'EOF' || true
%(cmd_options)s
EOF
{
echo -e "${cmd_options}" | diff -N -I'^\s*#' %(cmd_options_path)s -
} || {
@ -233,7 +239,6 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
'wrapper_path': wrapper_path,
'wrapper_dir': os.path.dirname(wrapper_path),
})
replace(context, "'", '"')
context.update({
'cmd_options': self.get_fcgid_cmd_options(webapp, context),
'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context,
@ -255,7 +260,5 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
'max_requests': settings.WEBAPPS_PHP_MAX_REQUESTS,
})
self.update_fpm_context(webapp, context)
# Fcgid context do contain special charactes
replace(context, "'", '"')
self.update_fcgid_context(webapp, context)
return context

13
orchestra/contrib/websites/backends/apache.py
View file

@ -5,7 +5,7 @@ import textwrap
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from .. import settings
@ -99,9 +99,11 @@ class Apache2Backend(ServiceController):
apache_conf += self.render_virtual_host(site, context, ssl=True)
if site.protocol == site.HTTPS_ONLY:
apache_conf += self.render_redirect_https(context)
context['apache_conf'] = apache_conf.replace("'", '"')
context['apache_conf'] = apache_conf
self.append(textwrap.dedent("""\
apache_conf='%(apache_conf)s'
read -r -d '' apache_conf << 'EOF' || true
%(apache_conf)s
EOF
{
echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s -
} || {
@ -376,7 +378,7 @@ class Apache2Backend(ServiceController):
}
if not context['ips']:
raise ValueError("WEBSITES_DEFAULT_IPS is empty.")
return replace(context, "'", '"')
return context
def set_content_context(self, content, context):
content_context = {
@ -385,7 +387,6 @@ class Apache2Backend(ServiceController):
'app_name': content.webapp.name,
'app_path': content.webapp.get_path(),
}
content_context = replace(content_context, "'", '"')
context.update(content_context)
@ -458,4 +459,4 @@ class Apache2Traffic(ServiceMonitor):
'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': site.pk,
}
return replace(context, "'", '"')
return context

8
orchestra/contrib/websites/backends/webalizer.py
View file

@ -3,7 +3,7 @@ import textwrap
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController, replace
from orchestra.contrib.orchestration import ServiceController
from .. import settings
@ -26,7 +26,9 @@ class WebalizerBackend(ServiceController):
if [[ ! -e %(webalizer_path)s/index.html ]]; then
echo 'Webstats are coming soon' > %(webalizer_path)s/index.html
fi
echo '%(webalizer_conf)s' > %(webalizer_conf_path)s
cat << 'EOF' > %(webalizer_conf_path)s
%(webalizer_conf)s
EOF
chown %(user)s:www-data %(webalizer_path)s
chmod g+xr %(webalizer_path)s
""") % context
@ -98,4 +100,4 @@ class WebalizerBackend(ServiceController):
SearchEngine alltheweb.com query=
DumpSites yes""") % context
return replace(context, "'", '"')
return context

3
orchestra/management/commands/setupnginx.py
View file

@ -152,6 +152,7 @@ class Command(BaseCommand):
'project_dir': paths.get_project_dir(),
'site_dir': paths.get_site_dir(),
'static_root': settings.STATIC_ROOT,
'static_url': (settings.STATIC_URL or '/static').rstrip('/')
'user': user,
'group': options.get('group') or user,
'home': expanduser("~%s" % options.get('user')),
@ -178,7 +179,7 @@ class Command(BaseCommand):
uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;
include uwsgi_params;
}
location /static {
location %(static_url)s {
alias %(static_root)s;
expires 30d;
}