diff --git a/TODO.md b/TODO.md index e1b7f242..951032e7 100644 --- a/TODO.md +++ b/TODO.md @@ -418,15 +418,19 @@ serailzer self.instance on create. # set_password serializer: "just-the-password" not {"password": "password"} -# use namedtuples! +# use namedtuples? # Negative transactionsx -# check certificate: websites directive ssl + domains search on miscellaneous +* check certificate: websites directive ssl + domains search on miscellaneous -# IF modsecurity... and Merge websites locations -# backend email error log with link to backend log on admin +# Merge websites locations # ValueError: Unable to configure handler 'file': [Errno 13] Permission denied: '/home/orchestra/panel/orchestra.log' +# billing invoice link on related invoices not overflow nginx GET vars + +* backendLog store method and language... and use it for display_script with correct lexer + +# process monitor data to represent state, or maybe create new resource datas when period expires? diff --git a/orchestra/admin/html.py b/orchestra/admin/html.py index c36322c6..d17cefd5 100644 --- a/orchestra/admin/html.py +++ b/orchestra/admin/html.py @@ -8,3 +8,13 @@ MONOSPACE_FONTS = ('Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans M def monospace_format(text): style="font-family:%s;padding-left:110px;" % MONOSPACE_FONTS return mark_safe('
%s
' % (style, text)) + + +def code_format(text, language='bash'): + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter + lexer = get_lexer_by_name(language, stripall=True) + formatter = HtmlFormatter(linenos=True) + code = highlight(text, lexer, formatter) + return mark_safe('
%s
' % code) diff --git a/orchestra/admin/utils.py b/orchestra/admin/utils.py index c9d118eb..0a70a89a 100644 --- a/orchestra/admin/utils.py +++ b/orchestra/admin/utils.py @@ -16,7 +16,7 @@ from orchestra.models.utils import get_field_value from orchestra.utils import humanize from .decorators import admin_field -from .html import monospace_format +from .html import monospace_format, code_format def get_modeladmin(model, import_module=True): @@ -165,3 +165,10 @@ def display_mono(field): return monospace_format(escape(getattr(log, field))) display.short_description = field return display + + +def display_code(field): + def display(self, log): + return code_format(getattr(log, field)) + display.short_description = field + return display diff --git a/orchestra/contrib/bills/actions.py b/orchestra/contrib/bills/actions.py index a73789af..43329f3a 100644 --- a/orchestra/contrib/bills/actions.py +++ b/orchestra/contrib/bills/actions.py @@ -9,6 +9,7 @@ from django.core.urlresolvers import reverse from django.db import transaction from django.http import HttpResponse from django.shortcuts import render, redirect +from django.utils import translation from django.utils.safestring import mark_safe from django.utils.translation import ungettext, ugettext_lazy as _ @@ -18,6 +19,7 @@ from orchestra.utils.html import html_to_pdf from .forms import SelectSourceForm from .helpers import validate_contact +from .models import Bill, BillLine def download_bills(modeladmin, request, queryset): @@ -209,3 +211,44 @@ def move_lines(modeladmin, request, queryset, action=None): def copy_lines(modeladmin, request, queryset): # same as move, but changing action behaviour return move_lines(modeladmin, request, queryset) + + +def amend_bills(modeladmin, request, queryset): + if queryset.filter(is_open=True).exists(): + messages.warning(request, _("Selected bills should be in closed state")) + return + ids = [] + for bill in queryset: + with translation.override(bill.account.language): + amend_type = bill.get_amend_type() + context = { + 'related_type': _(bill.get_type_display()), + 'number': bill.number, + 'date': bill.created_on, + } + amend = Bill.objects.create( + account=bill.account, + type=amend_type + ) + context['type'] = _(amend.get_type_display()) + amend.comments = _("%(type)s of %(related_type)s %(number)s and creation date %(date)s") % context + amend.save(update_fields=('comments',)) + for tax, subtotals in bill.compute_subtotals().items(): + context['tax'] = tax + line = BillLine.objects.create( + bill=amend, + start_on=bill.created_on, + description=_("Amend of %(related_type)s %(number)s, tax %(tax)s%%") % context, + subtotal=subtotals[0], + tax=tax + ) + ids.append(bill.pk) + amend_url = reverse('admin:bills_bill_changelist') + amend_url += '?id=%s' % ','.join(map(str, ids)) + messages.success(request, mark_safe(ungettext( + _('One amendment bill have been generated.') % amend_url, + _('%i amendment bills have been generated.') % (amend_url, len(ids)), + len(ids) + ))) +amend_bills.verbose_name = _("Amend") +amend_bills.url_name = 'amend' diff --git a/orchestra/contrib/bills/admin.py b/orchestra/contrib/bills/admin.py index b1d84684..3bbe9298 100644 --- a/orchestra/contrib/bills/admin.py +++ b/orchestra/contrib/bills/admin.py @@ -196,10 +196,11 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin): search_fields = ('number', 'account__username', 'comments') change_view_actions = [ actions.manage_lines, actions.view_bill, actions.download_bills, actions.send_bills, - actions.close_bills + actions.close_bills, actions.amend_bills, ] actions = [ - actions.manage_lines, actions.download_bills, actions.close_bills, actions.send_bills + actions.manage_lines, actions.download_bills, actions.close_bills, actions.send_bills, + actions.amend_bills, ] change_readonly_fields = ('account_link', 'type', 'is_open') readonly_fields = ('number', 'display_total', 'is_sent', 'display_payment_state') diff --git a/orchestra/contrib/bills/filters.py b/orchestra/contrib/bills/filters.py index 646ea2fe..a780567e 100644 --- a/orchestra/contrib/bills/filters.py +++ b/orchestra/contrib/bills/filters.py @@ -20,7 +20,7 @@ class BillTypeListFilter(SimpleListFilter): ('invoice', _("Invoice")), ('amendmentinvoice', _("Amendment invoice")), ('fee', _("Fee")), - ('fee', _("Amendment fee")), + ('amendmentfee', _("Amendment fee")), ('proforma', _("Pro-forma")), ) @@ -31,10 +31,11 @@ class BillTypeListFilter(SimpleListFilter): return self.request.path.split('/')[-2] def choices(self, cl): + query = self.request.GET.urlencode() for lookup, title in self.lookup_choices: yield { 'selected': self.value() == lookup, - 'query_string': reverse('admin:bills_%s_changelist' % lookup), + 'query_string': reverse('admin:bills_%s_changelist' % lookup) + '?%s' % query, 'display': title, } diff --git a/orchestra/contrib/bills/migrations/0004_auto_20150618_1311.py b/orchestra/contrib/bills/migrations/0004_auto_20150618_1311.py new file mode 100644 index 00000000..d3ec1ef8 --- /dev/null +++ b/orchestra/contrib/bills/migrations/0004_auto_20150618_1311.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bills', '0003_auto_20150612_0944'), + ] + + operations = [ + migrations.AlterField( + model_name='billcontact', + name='country', + field=models.CharField(max_length=20, verbose_name='country', default='ES', choices=[('HT', 'Haiti'), ('TJ', 'Tajikistan'), ('VG', 'Virgin Islands (British)'), ('NL', 'Netherlands'), ('BA', 'Bosnia and Herzegovina'), ('AT', 'Austria'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('UA', 'Ukraine'), ('GR', 'Greece'), ('VA', 'Holy See'), ('TW', 'Taiwan (Province of China)'), ('MD', 'Moldova (the Republic of)'), ('GE', 'Georgia'), ('MK', 'Macedonia (the former Yugoslav Republic of)'), ('IO', 'British Indian Ocean Territory'), ('HR', 'Croatia'), ('AS', 'American Samoa'), ('KE', 'Kenya'), ('CF', 'Central African Republic'), ('DM', 'Dominica'), ('AZ', 'Azerbaijan'), ('NF', 'Norfolk Island'), ('NZ', 'New Zealand'), ('ES', 'Spain'), ('GM', 'Gambia'), ('GD', 'Grenada'), ('FR', 'France'), ('SG', 'Singapore'), ('SO', 'Somalia'), ('AX', 'Åland Islands'), ('BT', 'Bhutan'), ('JP', 'Japan'), ('SK', 'Slovakia'), ('SY', 'Syrian Arab Republic'), ('VN', 'Viet Nam'), ('KM', 'Comoros'), ('SJ', 'Svalbard and Jan Mayen'), ('SC', 'Seychelles'), ('LV', 'Latvia'), ('RS', 'Serbia'), ('PY', 'Paraguay'), ('CZ', 'Czech Republic'), ('DO', 'Dominican Republic'), ('LI', 'Liechtenstein'), ('IN', 'India'), ('SS', 'South Sudan'), ('CC', 'Cocos (Keeling) Islands'), ('NC', 'New Caledonia'), ('HK', 'Hong Kong'), ('KW', 'Kuwait'), ('PM', 'Saint Pierre and Miquelon'), ('SB', 'Solomon Islands'), ('GA', 'Gabon'), ('PK', 'Pakistan'), ('QA', 'Qatar'), ('FJ', 'Fiji'), ('IS', 'Iceland'), ('SL', 'Sierra Leone'), ('CD', 'Congo (the Democratic Republic of the)'), ('BJ', 'Benin'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('MM', 'Myanmar'), ('TV', 'Tuvalu'), ('MY', 'Malaysia'), ('PR', 'Puerto Rico'), ('CI', "Côte d'Ivoire"), ('PF', 'French Polynesia'), ('GY', 'Guyana'), ('GU', 'Guam'), ('DK', 'Denmark'), ('UZ', 'Uzbekistan'), ('IR', 'Iran (Islamic Republic of)'), ('TC', 'Turks and Caicos Islands'), ('LU', 'Luxembourg'), ('UM', 'United States Minor Outlying Islands'), ('NG', 'Nigeria'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GL', 'Greenland'), ('VC', 'Saint Vincent and the Grenadines'), ('KN', 'Saint Kitts and Nevis'), ('PN', 'Pitcairn'), ('AR', 'Argentina'), ('CU', 'Cuba'), ('PL', 'Poland'), ('AL', 'Albania'), ('JE', 'Jersey'), ('PE', 'Peru'), ('PW', 'Palau'), ('IT', 'Italy'), ('AG', 'Antigua and Barbuda'), ('BR', 'Brazil'), ('MN', 'Mongolia'), ('MU', 'Mauritius'), ('TR', 'Turkey'), ('NE', 'Niger'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe'), ('VU', 'Vanuatu'), ('MP', 'Northern Mariana Islands'), ('TO', 'Tonga'), ('BD', 'Bangladesh'), ('NA', 'Namibia'), ('LS', 'Lesotho'), ('GP', 'Guadeloupe'), ('CY', 'Cyprus'), ('CA', 'Canada'), ('SN', 'Senegal'), ('CR', 'Costa Rica'), ('UG', 'Uganda'), ('CG', 'Congo'), ('MV', 'Maldives'), ('WS', 'Samoa'), ('MZ', 'Mozambique'), ('KR', 'Korea (the Republic of)'), ('TN', 'Tunisia'), ('LR', 'Liberia'), ('NU', 'Niue'), ('YE', 'Yemen'), ('EE', 'Estonia'), ('KG', 'Kyrgyzstan'), ('HN', 'Honduras'), ('KH', 'Cambodia'), ('RW', 'Rwanda'), ('BL', 'Saint Barthélemy'), ('UY', 'Uruguay'), ('SI', 'Slovenia'), ('GW', 'Guinea-Bissau'), ('DJ', 'Djibouti'), ('NI', 'Nicaragua'), ('AW', 'Aruba'), ('TH', 'Thailand'), ('IQ', 'Iraq'), ('BZ', 'Belize'), ('ZA', 'South Africa'), ('MC', 'Monaco'), ('BF', 'Burkina Faso'), ('KP', "Korea (the Democratic People's Republic of)"), ('BW', 'Botswana'), ('BM', 'Bermuda'), ('LB', 'Lebanon'), ('TG', 'Togo'), ('AU', 'Australia'), ('ER', 'Eritrea'), ('SE', 'Sweden'), ('AQ', 'Antarctica'), ('BH', 'Bahrain'), ('CW', 'Curaçao'), ('DZ', 'Algeria'), ('BS', 'Bahamas'), ('SX', 'Sint Maarten (Dutch part)'), ('CX', 'Christmas Island'), ('SH', 'Saint Helena, Ascension and Tristan da Cunha'), ('BN', 'Brunei Darussalam'), ('CV', 'Cabo Verde'), ('CH', 'Switzerland'), ('BV', 'Bouvet Island'), ('PS', 'Palestine, State of'), ('MT', 'Malta'), ('BB', 'Barbados'), ('BE', 'Belgium'), ('RU', 'Russian Federation'), ('GB', 'United Kingdom of Great Britain and Northern Ireland'), ('NO', 'Norway'), ('RE', 'Réunion'), ('AO', 'Angola'), ('CO', 'Colombia'), ('SA', 'Saudi Arabia'), ('LK', 'Sri Lanka'), ('ML', 'Mali'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('ME', 'Montenegro'), ('TM', 'Turkmenistan'), ('VI', 'Virgin Islands (U.S.)'), ('GF', 'French Guiana'), ('RO', 'Romania'), ('JM', 'Jamaica'), ('VE', 'Venezuela (Bolivarian Republic of)'), ('US', 'United States of America'), ('KI', 'Kiribati'), ('MX', 'Mexico'), ('SM', 'San Marino'), ('FM', 'Micronesia (Federated States of)'), ('GG', 'Guernsey'), ('LC', 'Saint Lucia'), ('MW', 'Malawi'), ('ET', 'Ethiopia'), ('BO', 'Bolivia (Plurinational State of)'), ('EC', 'Ecuador'), ('LT', 'Lithuania'), ('AI', 'Anguilla'), ('BY', 'Belarus'), ('AM', 'Armenia'), ('KZ', 'Kazakhstan'), ('TT', 'Trinidad and Tobago'), ('SR', 'Suriname'), ('MQ', 'Martinique'), ('GQ', 'Equatorial Guinea'), ('LY', 'Libya'), ('BG', 'Bulgaria'), ('CK', 'Cook Islands'), ('AD', 'Andorra'), ('DE', 'Germany'), ('MR', 'Mauritania'), ('TK', 'Tokelau'), ('EH', 'Western Sahara'), ('PT', 'Portugal'), ('CL', 'Chile'), ('GT', 'Guatemala'), ('KY', 'Cayman Islands'), ('SZ', 'Swaziland'), ('TL', 'Timor-Leste'), ('MF', 'Saint Martin (French part)'), ('SV', 'El Salvador'), ('EG', 'Egypt'), ('ST', 'Sao Tome and Principe'), ('HU', 'Hungary'), ('MA', 'Morocco'), ('TD', 'Chad'), ('PG', 'Papua New Guinea'), ('GN', 'Guinea'), ('MH', 'Marshall Islands'), ('FI', 'Finland'), ('BI', 'Burundi'), ('TZ', 'Tanzania, United Republic of'), ('CM', 'Cameroon'), ('FO', 'Faroe Islands'), ('TF', 'French Southern Territories'), ('FK', 'Falkland Islands [Malvinas]'), ('AF', 'Afghanistan'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('SD', 'Sudan'), ('HM', 'Heard Island and McDonald Islands'), ('OM', 'Oman'), ('LA', "Lao People's Democratic Republic"), ('WF', 'Wallis and Futuna'), ('MG', 'Madagascar'), ('AE', 'United Arab Emirates'), ('JO', 'Jordan'), ('PA', 'Panama'), ('IL', 'Israel'), ('YT', 'Mayotte'), ('IM', 'Isle of Man'), ('MO', 'Macao'), ('PH', 'Philippines'), ('MS', 'Montserrat'), ('CN', 'China')]), + ), + migrations.AlterField( + model_name='billline', + name='quantity', + field=models.DecimalField(verbose_name='quantity', decimal_places=2, max_digits=12, blank=True, null=True), + ), + ] diff --git a/orchestra/contrib/bills/models.py b/orchestra/contrib/bills/models.py index b0f9e2b9..cfefeb22 100644 --- a/orchestra/contrib/bills/models.py +++ b/orchestra/contrib/bills/models.py @@ -60,10 +60,17 @@ class BillManager(models.Manager): class Bill(models.Model): OPEN = '' + CREATED = 'CREATED' + PROCESSED = 'PROCESSED' + AMENDED = 'AMENDED' PAID = 'PAID' PENDING = 'PENDING' BAD_DEBT = 'BAD_DEBT' PAYMENT_STATES = ( + (OPEN, _("Open")), + (CREATED, _("Created")), + (PROCESSED, _("Processed")), + (AMENDED, _("Amended")), (PAID, _("Paid")), (PENDING, _("Pending")), (BAD_DEBT, _("Bad debt")), @@ -84,7 +91,8 @@ class Bill(models.Model): number = models.CharField(_("number"), max_length=16, unique=True, blank=True) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), - related_name='%(class)s') + related_name='%(class)s') +# amend_of = models.ForeignKey('self', null=True, blank=True, verbose_name=_("amend of"), related_name='amends') type = models.CharField(_("type"), max_length=16, choices=TYPES) created_on = models.DateField(_("created on"), auto_now_add=True) closed_on = models.DateField(_("closed on"), blank=True, null=True) @@ -125,6 +133,9 @@ class Bill(models.Model): def payment_state(self): if self.is_open or self.get_type() == self.PROFORMA: return self.OPEN +# elif self.amends.filter(is_open=False).exists(): +# return self.AMENDED + # TODO optimize this with a single query secured = self.transactions.secured().amount() or 0 if abs(secured) >= abs(self.get_total()): return self.PAID @@ -151,6 +162,16 @@ class Bill(models.Model): def get_type(self): return self.type or self.get_class_type() + def get_amend_type(self): + amend_map = { + self.INVOICE: self.AMENDMENTINVOICE, + self.FEE: self.AMENDMENTFEE, + } + amend_type = amend_map.get(self.type) + if amend_type is None: + raise TypeError("%s has no associated amend type." % self.type) + return amend_type + def get_number(self): cls = type(self) bill_type = self.get_type() @@ -298,7 +319,8 @@ class BillLine(models.Model): bill = models.ForeignKey(Bill, verbose_name=_("bill"), related_name='lines') description = models.CharField(_("description"), max_length=256) rate = models.DecimalField(_("rate"), blank=True, null=True, max_digits=12, decimal_places=2) - quantity = models.DecimalField(_("quantity"), max_digits=12, decimal_places=2) + quantity = models.DecimalField(_("quantity"), blank=True, null=True, max_digits=12, + decimal_places=2) verbose_quantity = models.CharField(_("Verbose quantity"), max_length=16) subtotal = models.DecimalField(_("subtotal"), max_digits=12, decimal_places=2) tax = models.DecimalField(_("tax"), max_digits=4, decimal_places=2) diff --git a/orchestra/contrib/bills/templates/bills/microspective.html b/orchestra/contrib/bills/templates/bills/microspective.html index e2bb421d..ee6c4b46 100644 --- a/orchestra/contrib/bills/templates/bills/microspective.html +++ b/orchestra/contrib/bills/templates/bills/microspective.html @@ -78,7 +78,7 @@
{% for line in lines %} {% with sublines=line.sublines.all description=line.description|slice:"38:" %} - {% if not line.order_id %}L{% endif %}{{ line.order_id }} + {% if not line.order_id %}L{% endif %}{{ line.order_id|default:line.pk }} {{ line.description|slice:":38" }} {{ line.get_verbose_period }} {{ line.get_verbose_quantity|default:" "|safe }} diff --git a/orchestra/contrib/mailboxes/admin.py b/orchestra/contrib/mailboxes/admin.py index 7240d5ef..c31c41dd 100644 --- a/orchestra/contrib/mailboxes/admin.py +++ b/orchestra/contrib/mailboxes/admin.py @@ -36,7 +36,10 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo 'name', 'account_link', 'display_filtering', 'display_addresses', 'display_active', ) list_filter = (IsActiveListFilter, HasAddressListFilter, 'filtering') - search_fields = ('account__username', 'account__short_name', 'account__full_name', 'name') + search_fields = ( + 'account__username', 'account__short_name', 'account__full_name', 'name', + 'addresses__name', 'addresses__domain__name', + ) add_fieldsets = ( (None, { 'fields': ('account_link', 'name', 'password1', 'password2', 'filtering'), @@ -111,6 +114,13 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo form.modeladmin = self return form + def get_search_results(self, request, queryset, search_term): + # Remove local domain from the search term if present (implicit local addreç) + search_term = search_term.replace('@'+settings.MAILBOXES_LOCAL_DOMAIN, '') + # Split address name from domain in order to support address searching + search_term = search_term.replace('@', ' ') + return super(MailboxAdmin, self).get_search_results(request, queryset, search_term) + def save_model(self, request, obj, form, change): """ save hacky mailbox.addresses """ super(MailboxAdmin, self).save_model(request, obj, form, change) diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 4de35c93..ed3ee3e1 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -240,13 +240,13 @@ class PostfixAddressVirtualDomainBackend(ServiceController): ('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH') ) - def is_local_domain(self, domain): + def is_hosted_domain(self, domain): """ whether or not domain MX points to this server """ return domain.has_default_mx() def include_virtual_alias_domain(self, context): domain = context['domain'] - if domain.name != context['local_domain'] and self.is_local_domain(domain): + if domain.name != context['local_domain'] and self.is_hosted_domain(domain): self.append(textwrap.dedent(""" # %(domain)s is a virtual domain belonging to this server if [[ ! $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]]; then diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index f1cf7608..42f88fc2 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -4,7 +4,7 @@ from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin -from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono +from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono, display_code from . import settings, helpers from .backends import ServiceBackend @@ -122,7 +122,7 @@ class BackendLogAdmin(admin.ModelAdmin): date_hierarchy = 'created_at' inlines = (BackendOperationInline,) fields = ( - 'backend', 'server_link', 'state', 'mono_script', 'mono_stdout', + 'backend', 'server_link', 'state', 'display_script', 'mono_stdout', 'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created', 'execution_time' ) @@ -131,11 +131,16 @@ class BackendLogAdmin(admin.ModelAdmin): server_link = admin_link('server') display_created = admin_date('created_at', short_description=_("Created")) display_state = admin_colored('state', colors=STATE_COLORS) - mono_script = display_mono('script') + display_script = display_code('script') mono_stdout = display_mono('stdout') mono_stderr = display_mono('stderr') mono_traceback = display_mono('traceback') + class Media: + css = { + 'all': ('orchestra/css/pygments/github.css',) + } + def get_queryset(self, request): """ Order by structured name and imporve performance """ qs = super(BackendLogAdmin, self).get_queryset(request) diff --git a/orchestra/contrib/orchestration/helpers.py b/orchestra/contrib/orchestration/helpers.py index 8370c086..e9c25839 100644 --- a/orchestra/contrib/orchestration/helpers.py +++ b/orchestra/contrib/orchestration/helpers.py @@ -61,8 +61,9 @@ def send_report(method, args, log): backend = method.__self__.__class__.__name__ subject = '[Orchestra] %s execution %s on %s' % (backend, log.state, server) separator = "\n%s\n\n" % ('~ '*40,) - print(log.operations.all()) operations = '\n'.join([' '.join((op.action, get_instance_url(op))) for op in log.operations.all()]) + log_url = reverse('admin:orchestration_backendlog_change', args=(log.pk,)) + log_url = orchestra_settings.ORCHESTRA_SITE_URL + log_url message = separator.join([ "[EXIT CODE] %s" % log.exit_code, "[STDERR]\n%s" % log.stderr, @@ -70,6 +71,7 @@ def send_report(method, args, log): "[SCRIPT]\n%s" % log.script, "[TRACEBACK]\n%s" % log.traceback, "[OPERATIONS]\n%s" % operations, + "[BACKEND LOG] %s" % log_url, ]) html_message = '\n\n'.join([ '

Exit code %s

' % log.exit_code, @@ -83,6 +85,7 @@ def send_report(method, args, log): '
%s
' % escape(log.traceback), '

Operations

' '
%s
' % escape(operations), + '

Backend log %s

' % (log_url, log_url), ]) mail_admins(subject, message, html_message=html_message) diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index ae3458d8..67a708ac 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -63,7 +63,11 @@ class Command(BaseCommand): server = Server(name=server, address=server) server.full_clean() server.save() - routes.append(AttrDict(host=server, async=False)) + routes.append(AttrDict( + host=server, + async=False, + action_is_async=lambda self: False, + )) # Generate operations for the given backend for instance in queryset: for backend in backends: @@ -79,7 +83,7 @@ class Command(BaseCommand): route, __, __ = key backend, operations = value servers.append(str(route.host)) - self.stdout.write('# Execute on %s' % route.host) + self.stdout.write('# Execute %s on %s' % (backend.get_name(), route.host)) for method, commands in backend.scripts: script = '\n'.join(commands) self.stdout.write(script) diff --git a/orchestra/contrib/payments/admin.py b/orchestra/contrib/payments/admin.py index 0e2f4eb1..8bff0a46 100644 --- a/orchestra/contrib/payments/admin.py +++ b/orchestra/contrib/payments/admin.py @@ -96,6 +96,7 @@ class TransactionAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): change_readonly_fields = ('amount', 'currency') readonly_fields = ('bill_link', 'display_state', 'process_link', 'account_link', 'source_link') list_select_related = ('source', 'bill__account') + date_hierarchy = 'created_at' bill_link = admin_link('bill') source_link = admin_link('source') diff --git a/orchestra/contrib/payments/models.py b/orchestra/contrib/payments/models.py index cdb98030..dc012d5a 100644 --- a/orchestra/contrib/payments/models.py +++ b/orchestra/contrib/payments/models.py @@ -76,7 +76,7 @@ class TransactionQuerySet(models.QuerySet): return self.exclude(state=Transaction.REJECTED) def amount(self): - return next(iter(self.aggregate(models.Sum('amount')).values())) + return next(iter(self.aggregate(models.Sum('amount')).values())) or 0 def processing(self): return self.filter(state__in=[Transaction.EXECUTED, Transaction.WAITTING_EXECUTION]) diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py index bce5f9cf..38b6f09e 100644 --- a/orchestra/contrib/webapps/backends/php.py +++ b/orchestra/contrib/webapps/backends/php.py @@ -141,17 +141,19 @@ class PHPBackend(WebAppServiceMixin, ServiceController): def prepare(self): super(PHPBackend, self).prepare() # Coordinate apache restart with php backend in order not to overdo it - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" backend="PHPBackend" - echo "$backend" >> /dev/shm/restart.apache2 - """) + echo "$backend" >> /dev/shm/restart.apache2""") ) def commit(self): + context = { + 'reload_pool': settings.WEBAPPS_PHPFPM_RELOAD_POOL, + } self.append(textwrap.dedent(""" # Apply changes if needed if [[ $UPDATED_FPM -eq 1 ]]; then - service php5-fpm reload + %(reload_pool)s fi # Coordinate Apache restart with other concurrent backends (e.g. Apache2Backend) @@ -182,7 +184,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): mv /dev/shm/restart.apache2.locked /dev/shm/restart.apache2 fi # End of coordination - """) + """) % context ) super(PHPBackend, self).commit() @@ -207,13 +209,10 @@ class PHPBackend(WebAppServiceMixin, ServiceController): pm = ondemand pm.max_requests = {{ max_requests }} pm.max_children = {{ max_children }} - - {% if request_terminate_timeout %}\ - request_terminate_timeout = {{ request_terminate_timeout }}\ - {% endif %} - {% for name, value in init_vars.items %}\ - php_admin_value[{{ name | safe }}] = {{ value | safe }}\ - {% endfor %} + {% if request_terminate_timeout %} + request_terminate_timeout = {{ request_terminate_timeout }}{% endif %} + {% for name, value in init_vars.items %} + php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %} """ )) return fpm_config.render(Context(context)) diff --git a/orchestra/contrib/webapps/backends/wordpress.py b/orchestra/contrib/webapps/backends/wordpress.py index 2462adcc..cc77b694 100644 --- a/orchestra/contrib/webapps/backends/wordpress.py +++ b/orchestra/contrib/webapps/backends/wordpress.py @@ -33,6 +33,9 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): echo "ERROR: execution returned non-zero code: $exit_code. cmd was:\\n$cmd\\n"; exit($exit_code); } + } + function wp_new_blog_notification($blog_title, $blog_url, $user_id, $password){ + // do nothing }""") ) @@ -113,9 +116,6 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): $_POST['admin_password'] = "%(password)s"; $_POST['admin_password2'] = "%(password)s"; - function wp_new_blog_notification($blog_title, $blog_url, $user_id, $password){ - // do nothing - } ob_start(); require_once('%(app_path)s/wp-admin/install.php'); $response = ob_get_contents(); diff --git a/orchestra/contrib/webapps/settings.py b/orchestra/contrib/webapps/settings.py index 5f673f9b..8f2b5363 100644 --- a/orchestra/contrib/webapps/settings.py +++ b/orchestra/contrib/webapps/settings.py @@ -35,6 +35,10 @@ WEBAPPS_PHPFPM_POOL_PATH = Setting('WEBAPPS_PHPFPM_POOL_PATH', validators=[Setting.string_format_validator(_php_names)], ) +WEBAPPS_PHPFPM_RELOAD_POOL = Setting('WEBAPPS_PHPFPM_RELOAD_POOL', + 'service php5-fpm reload' +) + WEBAPPS_FCGID_WRAPPER_PATH = Setting('WEBAPPS_FCGID_WRAPPER_PATH', '/home/httpd/fcgi-bin.d/%(user)s/%(app_name)s-wrapper', diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py index c86a672b..d7d93ec6 100644 --- a/orchestra/contrib/webapps/types/php.py +++ b/orchestra/contrib/webapps/types/php.py @@ -65,10 +65,11 @@ class PHPApp(AppType): 'webapp_id': self.instance.pk, } if merge: + php_version = self.instance.data.get('php_version', self.DEFAULT_PHP_VERSION) kwargs = { # webapp__type is not used because wordpress != php != symlink... 'webapp__account': self.instance.account_id, - 'webapp__data__contains': '"php_version":"%s"' % self.instance.data['php_version'], + 'webapp__data__contains': '"php_version":"%s"' % php_version, } return self.instance.get_options(**kwargs) diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py index 83deb1f5..7521bb8b 100644 --- a/orchestra/contrib/websites/backends/apache.py +++ b/orchestra/contrib/websites/backends/apache.py @@ -40,6 +40,7 @@ class Apache2Backend(ServiceController): def render_virtual_host(self, site, context, ssl=False): context['port'] = self.HTTPS_PORT if ssl else self.HTTP_PORT + context['vhost_wrapper_dirs'] = [] extra_conf = self.get_content_directives(site, context) directives = site.get_directives() if ssl: @@ -141,10 +142,9 @@ class Apache2Backend(ServiceController): def prepare(self): super(Apache2Backend, self).prepare() # Coordinate apache restart with php backend in order not to overdo it - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" backend="Apache2Backend" - echo "$backend" >> /dev/shm/restart.apache2\ - """) + echo "$backend" >> /dev/shm/restart.apache2""") ) def commit(self): @@ -187,8 +187,8 @@ class Apache2Backend(ServiceController): try: method = getattr(self, 'get_%s_directives' % method) except AttributeError: - raise AttributeError("%s does not has suport for '%s' directive." % - (self.__class__.__name__, method)) + context = (self.__class__.__name__, method) + raise AttributeError("%s does not has suport for '%s' directive." % context) return method(context, *args) def get_content_directives(self, site, context): @@ -238,10 +238,10 @@ class Apache2Backend(ServiceController): directives = '' # This Action trick is used instead of FcgidWrapper because we don't want to define # a new fcgid process class each time an app is mounted (num proc limits enforcement). - if 'wrapper_dir' not in context: + context['wrapper_dir'] = os.path.dirname(wrapper_path) + if context['wrapper_dir'] not in context['vhost_wrapper_dirs']: # fcgi-bin only needs to be defined once per vhots # We assume that all account wrapper paths will share the same dir - context['wrapper_dir'] = os.path.dirname(wrapper_path) directives = textwrap.dedent("""\ Alias /fcgi-bin/ %(wrapper_dir)s/ @@ -249,6 +249,7 @@ class Apache2Backend(ServiceController): Options +ExecCGI """) % context + context['vhost_wrapper_dirs'].append(context['wrapper_dir']) directives += self.get_location_filesystem_map(context) directives += textwrap.dedent(""" ProxyPass %(location)s/ ! @@ -279,26 +280,35 @@ class Apache2Backend(ServiceController): ca = [settings.WEBSITES_DEFAULT_SSL_CA] if not (cert and key): return [] - config = "SSLEngine on\n" - config += "SSLCertificateFile %s\n" % cert[0] - config += "SSLCertificateKeyFile %s\n" % key[0] + ssl_config = [ + "SSLEngine on", + "SSLCertificateFile %s" % cert[0], + "SSLCertificateKeyFile %s" % key[0], + ] if ca: - config += "SSLCACertificateFile %s\n" % ca[0] + ssl_config.append("SSLCACertificateFile %s" % ca[0]) return [ - ('', config), + ('', '\n'.join(ssl_config)), ] def get_security(self, directives): - security = [] + remove_rules = [] for values in directives.get('sec-rule-remove', []): for rule in values.split(): - sec_rule = "SecRuleRemoveById %i" % int(rule) - security.append(('', sec_rule)) + sec_rule = " SecRuleRemoveById %i" % int(rule) + remove_rules.append(sec_rule) + security = [] + if remove_rules: + remove_rules.insert(0, '') + remove_rules.append('') + security.append(('', '\n'.join(remove_rules))) for location in directives.get('sec-engine', []): sec_rule = textwrap.dedent("""\ - - SecRuleEngine off - """) % location + + + SecRuleEngine Off + + """) % location security.append((location, sec_rule)) return security @@ -466,9 +476,8 @@ class Apache2Traffic(ServiceMonitor): self.append('monitor {object_id} "{last_date}" {log_file}'.format(**context)) def get_context(self, site): - context = { + return { 'log_file': '%s{,.1}' % site.get_www_access_log_path(), 'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': site.pk, } - return context diff --git a/orchestra/contrib/websites/settings.py b/orchestra/contrib/websites/settings.py index a8ccfb23..71b4be97 100644 --- a/orchestra/contrib/websites/settings.py +++ b/orchestra/contrib/websites/settings.py @@ -105,9 +105,9 @@ WEBSITES_TRAFFIC_IGNORE_HOSTS = Setting('WEBSITES_TRAFFIC_IGNORE_HOSTS', WEBSITES_SAAS_DIRECTIVES = Setting('WEBSITES_SAAS_DIRECTIVES', { - 'wordpress-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock', '/home/httpd/wordpress-mu/'), - 'drupal-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/drupal-mu/'), - 'dokuwiki-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/moodle-mu/'), + 'wordpress-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock', '/home/httpd/wordpress-mu/'), + 'drupal-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock','/home/httpd/drupal-mu/'), + 'dokuwiki-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock','/home/httpd/moodle-mu/'), }, )