Fixes on webapps

TODO.md

@@ -283,6 +283,9 @@ https://code.djangoproject.com/ticket/24576
 * MultiCHoiceField proper serialization
 
 # Apache restart fails: detect if appache running, and execute start
-# PHP backend is retarded does not detect well the version
 # Change crons, create cron for deleted webapps and users
 * UNIFY PHP FPM settings name
+# virtualhost name name-account?
+# php version update should trigger webiste upgrade (wrapper name/fpm config for apache), public root and other config also needs apache to execute
+* add a delay to changes on the webserver apache to no overwelm it with backend executions?
+# Delete webapps deletes wrapper that may be used for other sites, maybe merging webapps is a bad idea after all?

orchestra/contrib/webapps/backends/php.py

@@ -76,27 +76,44 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         self.delete_webapp_dir(context)
     
     def delete_fpm(self, webapp, context):
-        self.append("rm -f %(fpm_path)s" % context)
+        # Better not delete a pool used by other apps
+        if not self.MERGE:
+            self.append("rm -f %(fpm_path)s" % context)
     
     def delete_fcgid(self, webapp, context):
-        self.append("rm -f %(wrapper_path)s" % context)
-        self.append("rm -f %(cmd_options_path)s" % context)
+        # Better not delete a wrapper used by other apps
+        if not self.MERGE:
+            self.append("rm -f %(wrapper_path)s" % context)
+            self.append("rm -f %(cmd_options_path)s" % context)
+    
+    def prepare(self):
+        super(PHPBackend, self).prepare()
+        # Coordinate apache restart with php backend in order not to overdo it
+        self.append('echo "PHPBackend" >> /dev/shm/restart.apache2')
     
     def commit(self):
-        if self.content:
-            self.append(textwrap.dedent("""
-                if [[ $UPDATEDFPM == 1 ]]; then
-                    service php5-fpm reload
-                    service php5-fpm start
-                fi
-                """)
-            )
-            self.append(textwrap.dedent("""\
-                if [[ $UPDATED_APACHE == 1 ]]; then
+        self.append(textwrap.dedent("""
+            if [[ $UPDATEDFPM == 1 ]]; then
+                service php5-fpm reload
+                service php5-fpm start
+            fi
+            # Coordinate apache restart with apache backend
+            locked=1
+            state="$(grep -v 'PHPBackend' /dev/shm/restart.apache2)" || locked=0
+            echo -n "$state" > /dev/shm/restart.apache2
+            if [[ $UPDATED_APACHE == 1 ]]; then
+                if [[ $locked == 0 ]]; then
                     service apache2 reload
+                else
+                    echo "PHPBackend RESTART" >> /dev/shm/restart.apache2
                 fi
-                """)
-            )
+            elif [[ "$state" =~ .*RESTART$ ]]; then
+                rm /dev/shm/restart.apache2
+                service apache2 reload
+            fi
+            """)
+        )
+        super(PHPBackend, self).commit()
     
     def get_fpm_config(self, webapp, context):
         merge = settings.WEBAPPS_MERGE_PHP_WEBAPPS

orchestra/contrib/webapps/models.py

@@ -1,4 +1,5 @@
 import os
+from collections import OrderedDict
 
 from django.db import models
 from django.db.models.signals import pre_save, pre_delete
@@ -56,9 +57,7 @@ class WebApp(models.Model):
     
     @cached
     def get_options(self):
-        return {
-            opt.name: opt.value for opt in self.options.all()
-        }
+        return OrderedDict((opt.name, opt.value) for opt in self.options.all())
     
     def get_directive(self):
         return self.type_instance.get_directive()

orchestra/contrib/webapps/options.py

@@ -56,7 +56,7 @@ class PHPAppOption(AppOption):
         super(PHPAppOption, self).validate()
         if self.deprecated:
             php_version = self.instance.webapp.type_instance.get_php_version_number()
-            if php_version and php_version > self.deprecated:
+            if php_version and self.deprecated and float(php_version) > self.deprecated:
                 raise ValidationError(
                     _("This option is deprecated since PHP version %s.") % str(self.deprecated)
                 )
@@ -76,7 +76,8 @@ class Timeout(AppOption):
     # FPM pm.request_terminate_timeout
     # PHP max_execution_time ini
     verbose_name = _("Process timeout")
-    help_text = _("Maximum time in seconds allowed for a request to complete (a number between 0 and 999).")
+    help_text = _("Maximum time in seconds allowed for a request to complete (a number between 0 and 999).<br>"
+                  "Also sets <tt>max_request_time</tt> when php-cgi is used.")
     regex = r'^[0-9]{1,3}$'
     group = AppOption.PROCESS
 
@@ -94,7 +95,7 @@ class Processes(AppOption):
 class PHPEnabledFunctions(PHPAppOption):
     name = 'enabled_functions'
     verbose_name = _("Enabled functions")
-    help_text = ' '.join(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS)
+    help_text = ','.join(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS)
     regex = r'^[\w\.,-]+$'
 
 
@@ -145,7 +146,7 @@ class PHPDisplayErrors(PHPAppOption):
     name = 'display_errors'
     verbose_name = _("Display errors")
     help_text = _("Determines whether errors should be printed to the screen as part of the output or "
-                "if they should be hidden from the user (On or Off).")
+                  "if they should be hidden from the user (On or Off).")
     regex = r'^(On|Off|on|off)$'
 
 
@@ -159,7 +160,7 @@ class PHPMagicQuotesGPC(PHPAppOption):
     name = 'magic_quotes_gpc'
     verbose_name = _("Magic quotes GPC")
     help_text = _("Sets the magic_quotes state for GPC (Get/Post/Cookie) operations (On or Off) "
-                "<b>DEPRECATED as of PHP 5.3.0</b>.")
+                  "<b>DEPRECATED as of PHP 5.3.0</b>.")
     regex = r'^(On|Off|on|off)$'
     deprecated = 5.3
 
@@ -168,7 +169,7 @@ class PHPMagicQuotesRuntime(PHPAppOption):
     name = 'magic_quotes_runtime'
     verbose_name = _("Magic quotes runtime")
     help_text = _("Functions that return data from any sort of external source will have quotes escaped "
-                "with a backslash (On or Off) <b>DEPRECATED as of PHP 5.3.0</b>.")
+                  "with a backslash (On or Off) <b>DEPRECATED as of PHP 5.3.0</b>.")
     regex = r'^(On|Off|on|off)$'
     deprecated = 5.3
 
@@ -200,7 +201,7 @@ class PHPMemoryLimit(PHPAppOption):
     name = 'memory_limit'
     verbose_name = _("Memory limit")
     help_text = _("This sets the maximum amount of memory in bytes that a script is allowed to allocate "
-                "(Value between 0M and 999M).")
+                  "(Value between 0M and 999M).")
     regex = r'^[0-9]{1,3}M$'
 
 
@@ -222,7 +223,7 @@ class PHPRegisterGlobals(PHPAppOption):
     name = 'register_globals'
     verbose_name = _("Register globals")
     help_text = _("Whether or not to register the EGPCS (Environment, GET, POST, Cookie, Server) "
-                "variables as global variables (On or Off).")
+                  "variables as global variables (On or Off).")
     regex = r'^(On|Off|on|off)$'
 
 
@@ -235,21 +236,21 @@ class PHPPostMaxSize(PHPAppOption):
 
 class PHPSendmailPath(PHPAppOption):
     name = 'sendmail_path'
-    verbose_name = _("sendmail_path")
+    verbose_name = _("Sendmail path")
     help_text = _("Where the sendmail program can be found.")
     regex = r'^[^ ]+$'
 
 
 class PHPSessionBugCompatWarn(PHPAppOption):
     name = 'session.bug_compat_warn'
-    verbose_name = _("session.bug_compat_warn")
+    verbose_name = _("Session bug compat warning")
     help_text = _("Enables an PHP bug on session initialization for legacy behaviour (On or Off).")
     regex = r'^(On|Off|on|off)$'
 
 
 class PHPSessionAutoStart(PHPAppOption):
     name = 'session.auto_start'
-    verbose_name = _("session.auto_start")
+    verbose_name = _("Session auto start")
     help_text = _("Specifies whether the session module starts a session automatically on request "
                 "startup (On or Off).")
     regex = r'^(On|Off|on|off)$'
@@ -287,7 +288,7 @@ class PHPSuhosinRequestMaxVars(PHPAppOption):
 
 class PHPSuhosinSessionEncrypt(PHPAppOption):
     name = 'suhosin.session.encrypt'
-    verbose_name = _("suhosin.session.encrypt")
+    verbose_name = _("Suhosin session encrypt")
     help_text = _("On or Off")
     regex = r'^(On|Off|on|off)$'
 
@@ -301,13 +302,13 @@ class PHPSuhosinSimulation(PHPAppOption):
 
 class PHPSuhosinExecutorIncludeWhitelist(PHPAppOption):
     name = 'suhosin.executor.include.whitelist'
-    verbose_name = _("suhosin.executor.include.whitelist")
+    verbose_name = _("Suhosin executor include whitelist")
     regex = r'.*$'
 
 
 class PHPUploadMaxFileSize(PHPAppOption):
     name = 'upload_max_filesize'
-    verbose_name = _("upload_max_filesize")
+    verbose_name = _("Upload max filezise")
     help_text = _("Value between 0M and 999M.")
     regex = r'^[0-9]{1,3}M$'
 

orchestra/contrib/webapps/settings.py

@@ -110,6 +110,7 @@ WEBAPPS_UNDER_CONSTRUCTION_PATH = getattr(settings, 'WEBAPPS_UNDER_CONSTRUCTION_
 #        WEBAPPS_TYPES[webapp_type] = value
 
 
+
 WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [
     'exec',
     'passthru',

orchestra/contrib/webapps/types/php.py

@@ -1,5 +1,6 @@
 import os
 import re
+from collections import OrderedDict
 
 from django import forms
 from django.utils.translation import ugettext_lazy as _
@@ -60,16 +61,16 @@ class PHPApp(AppType):
     
     @cached
     def get_php_options(self):
-        php_version = self.get_php_version_number()
+        php_version = float(self.get_php_version_number())
         php_options = AppOption.get_option_groups()[AppOption.PHP]
-        return [op for op in php_options if getattr(self, 'deprecated', 999) > php_version]
+        return [op for op in php_options if (op.deprecated or 999) > php_version]
     
     def get_php_init_vars(self, merge=False):
         """
         process php options for inclusion on php.ini
         per_account=True merges all (account, webapp.type) options
         """
-        init_vars = {}
+        init_vars = OrderedDict()
         options = self.instance.options.all()
         if merge:
             # Get options from the same account and php_version webapps
@@ -108,6 +109,7 @@ class PHPApp(AppType):
         context = super(PHPApp, self).get_directive_context()
         context.update({
             'php_version': self.get_php_version(),
+            'php_version_number': self.get_php_version_number(),
         })
         return context
     
@@ -134,4 +136,4 @@ class PHPApp(AppType):
             raise ValueError("No version number matches for '%s'" % php_version)
         if len(number) > 1:
             raise ValueError("Multiple version number matches for '%s'" % php_version)
-        return float(number[0])
+        return number[0]

orchestra/contrib/websites/backends/apache.py

@@ -19,6 +19,7 @@ class Apache2Backend(ServiceController):
     model = 'websites.Website'
     related_models = (
         ('websites.Content', 'website'),
+        ('webapps.WebApp', 'website_set'),
     )
     verbose_name = _("Apache 2")
     
@@ -41,9 +42,9 @@ class Apache2Backend(ServiceController):
         return Template(textwrap.dedent("""\
             <VirtualHost {{ ip }}:{{ port }}>
                 IncludeOptional /etc/apache2/site[s]-override/{{ site_unique_name }}.con[f]
-                ServerName {{ site.domains.all|first }}\
-            {% if site.domains.all|slice:"1:" %}
-                ServerAlias {{ site.domains.all|slice:"1:"|join:' ' }}{% endif %}\
+                ServerName {{ server_name }}\
+            {% if server_alias %}
+                ServerAlias {{ server_alias|join:' ' }}{% endif %}\
             {% if access_log %}
                 CustomLog {{ access_log }} common{% endif %}\
             {% if error_log %}
@@ -59,9 +60,9 @@ class Apache2Backend(ServiceController):
         context['port'] = self.HTTP_PORT
         return Template(textwrap.dedent("""
             <VirtualHost {{ ip }}:{{ port }}>
-                ServerName {{ site.domains.all|first }}\
-            {% if site.domains.all|slice:"1:" %}
-                ServerAlias {{ site.domains.all|slice:"1:"|join:' ' }}{% endif %}\
+                ServerName {{ server_name }}\
+            {% if server_alias %}
+                ServerAlias {{ server_alias|join:' ' }}{% endif %}\
             {% if access_log %}
                 CustomLog {{ access_log }} common{% endif %}\
             {% if error_log %}
@@ -75,33 +76,67 @@ class Apache2Backend(ServiceController):
     
     def save(self, site):
         context = self.get_context(site)
-        apache_conf = '# %(banner)s\n' % context
-        if site.protocol in (site.HTTP, site.HTTP_AND_HTTPS):
-            apache_conf += self.render_virtual_host(site, context, ssl=False)
-        if site.protocol in (site.HTTP_AND_HTTPS, site.HTTPS_ONLY, site.HTTPS):
-            apache_conf += self.render_virtual_host(site, context, ssl=True)
-        if site.protocol == site.HTTPS_ONLY:
-            apache_conf += self.render_redirect_https(context)
-        context['apache_conf'] = apache_conf.replace("'", '"')
-        self.append(textwrap.dedent("""\
-            apache_conf='%(apache_conf)s'
-            {
-                echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s -
-            } || {
-                echo -e "${apache_conf}" > %(sites_available)s
-                UPDATED=1
-            }""") % context
-        )
-        self.enable_or_disable(site)
+        if context['server_name']:
+            apache_conf = '# %(banner)s\n' % context
+            if site.protocol in (site.HTTP, site.HTTP_AND_HTTPS):
+                apache_conf += self.render_virtual_host(site, context, ssl=False)
+            if site.protocol in (site.HTTP_AND_HTTPS, site.HTTPS_ONLY, site.HTTPS):
+                apache_conf += self.render_virtual_host(site, context, ssl=True)
+            if site.protocol == site.HTTPS_ONLY:
+                apache_conf += self.render_redirect_https(context)
+            context['apache_conf'] = apache_conf.replace("'", '"')
+            self.append(textwrap.dedent("""\
+                apache_conf='%(apache_conf)s'
+                {
+                    echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s -
+                } || {
+                    echo -e "${apache_conf}" > %(sites_available)s
+                    UPDATED=1
+                }""") % context
+            )
+        if context['server_name'] and site.active:
+            self.append(textwrap.dedent("""\
+                if [[ ! -f %(sites_enabled)s ]]; then
+                    a2ensite %(site_unique_name)s.conf
+                    UPDATED=1
+                fi""") % context
+            )
+        else:
+            self.append(textwrap.dedent("""\
+                if [[ -f %(sites_enabled)s ]]; then
+                    a2dissite %(site_unique_name)s.conf;
+                    UPDATED=1
+                fi""") % context
+            )
     
     def delete(self, site):
         context = self.get_context(site)
         self.append("a2dissite %(site_unique_name)s.conf && UPDATED=1" % context)
         self.append("rm -f %(sites_available)s" % context)
     
+    def prepare(self):
+        super(Apache2Backend, self).prepare()
+        # Coordinate apache restart with php backend in order not to overdo it
+        self.append('echo "Apache2Backend" >> /dev/shm/restart.apache2')
+    
     def commit(self):
         """ reload Apache2 if necessary """
-        self.append('if [[ $UPDATED == 1 ]]; then service apache2 reload; fi')
+        self.append(textwrap.dedent("""\
+            locked=1
+            state="$(grep -v 'Apache2Backend' /dev/shm/restart.apache2)" || locked=0
+            echo -n "$state" > /dev/shm/restart.apache2
+            if [[ $UPDATED == 1 ]]; then
+                if [[ $locked == 0 ]]; then
+                    service apache2 reload
+                else
+                    echo "Apache2Backend RESTART" >> /dev/shm/restart.apache2
+                fi
+            elif [[ "$state" =~ .*RESTART$ ]]; then
+                rm /dev/shm/restart.apache2
+                service apache2 reload
+            fi""")
+        )
+        super(Apache2Backend, self).commit()
     
     def get_directives(self, directive, context):
         method, args = directive[0], directive[1:]
@@ -122,9 +157,15 @@ class Apache2Backend(ServiceController):
     
     def get_static_directives(self, context, app_path):
         context['app_path'] = os.path.normpath(app_path % context)
-        location = "%(location)s/" % context
-        directive = "Alias %(location)s/ %(app_path)s/" % context
-        return [(location, directive)]
+        directive = self.get_location_filesystem_map(context)
+        return [
+            (context['location'], directive),
+        ]
+    
+    def get_location_filesystem_map(self, context):
+        if not context['location']:
+            return 'DocumentRoot %(app_path)s' % context
+        return 'Alias %(location)s %(app_path)s' % context
     
     def get_fpm_directives(self, context, socket, app_path):
         if ':' in socket:
@@ -139,28 +180,28 @@ class Apache2Backend(ServiceController):
             'app_path': os.path.normpath(app_path),
             'socket': socket,
         })
-        location = "%(location)s/" % context
-        directives = textwrap.dedent("""\
-            ProxyPassMatch ^%(location)s/(.*\.php(/.*)?)$ {target}
-            Alias %(location)s/ %(app_path)s/""".format(target=target) % context
-        )
-        return [(location, directives)]
+        directives = "ProxyPassMatch ^%(location)s/(.*\.php(/.*)?)$ {target}\n".format(target=target) % context
+        directives += self.get_location_filesystem_map(context)
+        return [
+            (context['location'], directives),
+        ]
     
     def get_fcgid_directives(self, context, app_path, wrapper_path):
         context.update({
             'app_path': os.path.normpath(app_path),
             'wrapper_path': wrapper_path,
         })
-        location = "%(location)s/" % context
-        directives = textwrap.dedent("""\
-            Alias %(location)s/ %(app_path)s/
+        directives = self.get_location_filesystem_map(context)
+        directives += textwrap.dedent("""
             ProxyPass %(location)s/ !
             <Directory %(app_path)s/>
                 Options +ExecCGI
                 AddHandler fcgid-script .php
                 FcgidWrapper %(wrapper_path)s
             </Directory>""") % context
-        return [(location, directives)]
+        return [
+            (context['location'], directives),
+        ]
     
     def get_ssl(self, directives):
         cert = directives.get('ssl-cert')
@@ -177,7 +218,9 @@ class Apache2Backend(ServiceController):
         config += "SSLCertificateKeyFile %s\n" % key[0]
         if ca:
            config += "SSLCACertificateFile %s\n" % ca[0]
-        return [('', config)]
+        return [
+            ('', config),
+        ]
         
     def get_security(self, directives):
         security = []
@@ -201,20 +244,27 @@ class Apache2Backend(ServiceController):
                 redirect = "RedirectMatch %s %s" % (location, target)
             else:
                 redirect = "Redirect %s %s" % (location, target)
-            redirects.append((location, redirect))
+            redirects.append(
+                (location, redirect)
+            )
         return redirects
     
     def get_proxies(self, directives):
         proxies = []
         for proxy in directives.get('proxy', []):
-            location, target = proxy.split()
+            proxy = proxy.split()
+            location = proxy[0]
+            target = proxy[1]
+            options = ' '.join(proxy[2:])
             location = normurlpath(location)
             proxy = textwrap.dedent("""\
-                ProxyPass {location}/ {target}
+                ProxyPass {location}/ {target} {options}
                 ProxyPassReverse {location}/ {target}""".format(
-                    location=location, target=target)
+                    location=location, target=target, options=options)
+            )
+            proxies.append(
+                (location, proxy)
             )
-            proxies.append((location, proxy))
         return proxies
     
     def get_saas(self, directives):
@@ -229,23 +279,6 @@ class Apache2Backend(ServiceController):
                     saas += self.get_directives(directive, context)
         return saas
     
-    def enable_or_disable(self, site):
-        context = self.get_context(site)
-        if site.is_active:
-            self.append(textwrap.dedent("""\
-                if [[ ! -f %(sites_enabled)s ]]; then
-                    a2ensite %(site_unique_name)s.conf
-                    UPDATED=1
-                fi""") % context
-            )
-        else:
-            self.append(textwrap.dedent("""\
-                if [[ -f %(sites_enabled)s ]]; then
-                    a2dissite %(site_unique_name)s.conf;
-                    UPDATED=1
-                fi""") % context
-            )
-    
     def get_username(self, site):
         option = site.get_directives().get('user_group')
         if option:
@@ -259,10 +292,21 @@ class Apache2Backend(ServiceController):
             return group
         return site.get_groupname()
     
+    def get_server_names(self, site):
+        server_name = None
+        server_alias = []
+        for domain in site.domains.all():
+            if not server_name and not domain.name.startswith('*'):
+                server_name = domain.name
+            else:
+                server_alias.append(domain.name)
+        return server_name, server_alias
+    
     def get_context(self, site):
         base_apache_conf = settings.WEBSITES_BASE_APACHE_CONF
         sites_available = os.path.join(base_apache_conf, 'sites-available')
         sites_enabled = os.path.join(base_apache_conf, 'sites-enabled')
+        server_name, server_alias = self.get_server_names(site)
         context = {
             'site': site,
             'site_name': site.name,
@@ -270,6 +314,8 @@ class Apache2Backend(ServiceController):
             'site_unique_name': '0-'+site.unique_name,
             'user': self.get_username(site),
             'group': self.get_groupname(site),
+            'server_name': server_name,
+            'server_alias': server_alias,
             # TODO remove '0-'
             'sites_enabled': "%s.conf" % os.path.join(sites_enabled, '0-'+site.unique_name),
             'sites_available': "%s.conf" % os.path.join(sites_available, '0-'+site.unique_name),