More robust bash backends using heredoc

2015-05-21 17:53:59 +00:00 · 2015-05-21 17:53:59 +00:00 · 43eb744f81
parent f60376ab1d
commit 43eb744f81
11 changed files with 113 additions and 77 deletions
--- a/TODO.md
+++ b/TODO.md
@ -382,3 +382,5 @@ http://wiki2.dovecot.org/Pigeonhole/Sieve/Examples


 # mail system users group? which one is more convinient? if main group does not exists, backend will fail!
+
+Bash/Python/PHPBackend
--- a/orchestra/contrib/databases/backends.py
+++ b/orchestra/contrib/databases/backends.py
@ -23,20 +23,21 @@ class MySQLBackend(ServiceController):
        if database.type != database.MYSQL:
            return
        context = self.get_context(database)
-        self.append(
-            "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
-        )
        # Not available on delete()
        context['owner'] = database.owner
-        # clean previous privileges
-        self.append("""mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'""" % context)
+        self.append(textwrap.dedent("""
+            # Create database and re-set permissions
+            mysql -e 'CREATE DATABASE `%(database)s`;' || true
+            mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\
+            """) % context
+        )
        for user in database.users.all():
            context.update({
                'username': user.username,
                'grant': 'WITH GRANT OPTION' if user == context['owner'] else ''
            })
            self.append(textwrap.dedent("""\
-                mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;' \
+                mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;'\
                """) % context
            )
    
@ -44,11 +45,19 @@ class MySQLBackend(ServiceController):
        if database.type != database.MYSQL:
            return
        context = self.get_context(database)
-        self.append("mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=$?" % context)
-        self.append("mysql mysql -e 'DELETE FROM db WHERE db = \"%(database)s\";'" % context)
-        
+        self.append(textwrap.dedent("""
+            # Remove database %(database)s
+            mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=$?
+            mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\
+            """) % context
+        )
+    
    def commit(self):
-        self.append("mysql -e 'FLUSH PRIVILEGES;'")
+        self.append(textwrap.dedent("""
+            # Apply permissions
+            mysql -e 'FLUSH PRIVILEGES;'\
+            """)
+        )
        super(MySQLBackend, self).commit()
    
    def get_context(self, database):
@ -75,11 +84,9 @@ class MySQLUserBackend(ServiceController):
            return
        context = self.get_context(user)
        self.append(textwrap.dedent("""\
-            mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \
-            """) % context
-        )
-        self.append(textwrap.dedent("""\
-            mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \
+            # Create user %(username)s
+            mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true
+            mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";'\
            """) % context
        )
    
@ -87,12 +94,14 @@ class MySQLUserBackend(ServiceController):
        if user.type != user.MYSQL:
            return
        context = self.get_context(user)
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
+            # Delete user %(username)s
            mysql -e 'DROP USER "%(username)s"@"%(host)s";' || exit_code=$? \
            """) % context
        )
    
    def commit(self):
+        self.append("# Apply permissions")
        self.append("mysql -e 'FLUSH PRIVILEGES;'")
    
    def get_context(self, user):
--- a/orchestra/contrib/domains/backends.py
+++ b/orchestra/contrib/domains/backends.py
@ -45,19 +45,21 @@ class Bind9MasterDomainBackend(ServiceController):
    def update_zone(self, domain, context):
        context['zone'] = ';; %(banner)s\n' % context
        context['zone'] += domain.render_zone()
-        self.append(textwrap.dedent("""
+        self.append(textwrap.dedent("""\
+            # Generate %(name)s zone file
            cat << 'EOF' > %(zone_path)s.tmp
            %(zone)s
            EOF
            diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1
            # Because bind reload will not display any fucking error
            named-checkzone -k fail -n fail %(name)s %(zone_path)s.tmp
-            mv %(zone_path)s.tmp %(zone_path)s
+            mv %(zone_path)s.tmp %(zone_path)s\
            """) % context
        )
    
    def update_conf(self, context):
        self.append(textwrap.dedent("""
+            # Update bind config file for %(name)s
            read -r -d '' conf << 'EOF' || true
            %(conf)s
            EOF
@ -68,8 +70,8 @@ class Bind9MasterDomainBackend(ServiceController):
                UPDATED=1
            }""") % context
        )
-        # Delete ex-top-domains that are now subdomains
        self.append(textwrap.dedent("""\
+            # Delete ex-top-domains that are now subdomains
            sed -i -e '/zone\s\s*".*\.%(name)s".*/,/^\s*};\s*$/d' \\
                   -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s""") % context
        )
@ -79,6 +81,7 @@ class Bind9MasterDomainBackend(ServiceController):
    
    def delete(self, domain):
        context = self.get_context(domain)
+        self.append('# Delete zone file for %(name)s' % context)
        self.append('rm -f %(zone_path)s;' % context)
        self.delete_conf(context)
    
@ -87,6 +90,7 @@ class Bind9MasterDomainBackend(ServiceController):
            # These can never be top level domains
            return
        self.append(textwrap.dedent("""
+            # Delete config for %(name)s
            sed -e '/zone\s\s*"%(name)s".*/,/^\s*};\s*$/d' \\
                -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s > %(conf_path)s.tmp""") % context
        )
@ -95,7 +99,12 @@ class Bind9MasterDomainBackend(ServiceController):
    
    def commit(self):
        """ reload bind if needed """
-        self.append('if [[ $UPDATED == 1 ]]; then service bind9 reload; fi')
+        self.append(textwrap.dedent("""
+            # Apply changes
+            if [[ $UPDATED == 1 ]]; then
+                service bind9 reload
+            fi""")
+        )
    
    def get_servers(self, domain, backend):
        """ Get related server IPs from registered backend routes """
@ -180,12 +189,12 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
        self.delete_conf(context)
    
    def commit(self):
-        """ ideally slave should be restarted after master """
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
+            # Apply changes
            if [[ $UPDATED == 1 ]]; then
+                # Async restart, ideally after master
                nohup bash -c 'sleep 1 && service bind9 reload' &> /dev/null &
-            fi
-            """)
+            fi""")
        )
    
    def get_context(self, domain):
@ -196,7 +205,7 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
            'masters': '; '.join(self.get_masters_ips(domain)) or 'none',
            'conf_path': self.CONF_PATH,
        }
-        context['conf'] = textwrap.dedent("""
+        context['conf'] = textwrap.dedent("""\
            zone "%(name)s" {
                // %(banner)s
                type slave;
--- a/orchestra/contrib/lists/backends.py
+++ b/orchestra/contrib/lists/backends.py
@ -27,6 +27,7 @@ class MailmanVirtualDomainBackend(ServiceController):
        domain = context['address_domain']
        if domain and self.is_local_domain(domain):
            self.append(textwrap.dedent("""
+                # Add virtual domain %(address_domain)s
                [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || {
                    echo '%(address_domain)s' >> %(virtual_alias_domains)s
                    UPDATED_VIRTUAL_ALIAS_DOMAINS=1
@ -39,7 +40,11 @@ class MailmanVirtualDomainBackend(ServiceController):
    def exclude_virtual_alias_domain(self, context):
        domain = context['address_domain']
        if domain and self.is_last_domain(domain):
-            self.append("sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s" % context)
+            self.append(textwrap.dedent("""
+                # Remove %(address_domain)s from virtual domains
+                sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s\
+                """) % context
+            )
    
    def save(self, mail_list):
        context = self.get_context(mail_list)
@ -52,10 +57,12 @@ class MailmanVirtualDomainBackend(ServiceController):
    def commit(self):
        context = self.get_context_files()
        self.append(textwrap.dedent("""
+            # Apply changes if needed
            if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then
                service postfix reload
            fi""") % context
        )
+        super(MailmanVirtualDomainBackend, self).commit()
    
    def get_context_files(self):
        return {
@ -108,15 +115,16 @@ class MailmanBackend(MailmanVirtualDomainBackend):
    def save(self, mail_list):
        context = self.get_context(mail_list)
        # Create list
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
+            # Create list %(name)s
            [[ ! -e '%(mailman_root)s/lists/%(name)s' ]] && {
                newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s'
            }""") % context)
        # Custom domain
        if mail_list.address:
            context['aliases'] = self.get_virtual_aliases(context)
-            # Preserve indentation
            self.append(textwrap.dedent("""\
+                # Create list alias for custom domain
                aliases='%(aliases)s'
                if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then
                    echo "${aliases}" >> %(virtual_alias)s
@ -128,27 +136,25 @@ class MailmanBackend(MailmanVirtualDomainBackend):
                        echo "${aliases}" >> %(virtual_alias)s
                        UPDATED_VIRTUAL_ALIAS=1
                    fi
-                fi""") % context
-            )
-            self.append(
-                'echo "require_explicit_destination = 0" | '
-                '%(mailman_root)s/bin/config_list -i /dev/stdin %(name)s' % context
-            )
-            self.append(textwrap.dedent("""\
-                echo "host_name = '%(address_domain)s'" | \
+                fi
+                echo "require_explicit_destination = 0" | \\
+                    %(mailman_root)s/bin/config_list -i /dev/stdin %(name)s
+                echo "host_name = '%(address_domain)s'" | \\
                    %(mailman_root)s/bin/config_list -i /dev/stdin %(name)s""") % context
            )
        else:
-            # Cleanup shit
            self.append(textwrap.dedent("""\
+                # Cleanup possible ex-custom domain
                if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then
                    sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s
                fi""") % context
            )
        # Update
        if context['password'] is not None:
-            self.append(
-                '%(mailman_root)s/bin/change_pw --listname="%(name)s" --password="%(password)s"' % context
+            self.append(textwrap.dedent("""\
+                # Re-set password
+                %(mailman_root)s/bin/change_pw --listname="%(name)s" --password="%(password)s"\
+                """) % context
            )
        self.include_virtual_alias_domain(context)
        if mail_list.active:
@ -160,10 +166,9 @@ class MailmanBackend(MailmanVirtualDomainBackend):
        context = self.get_context(mail_list)
        self.exclude_virtual_alias_domain(context)
        self.append(textwrap.dedent("""
+            # Remove list %(name)s
            sed -i -e '/^.*\s%(name)s\(%(suffixes_regex)s\)\s*$/d' \\
-                   -e 'N; /^\s*\\n\s*$/d; P; D' %(virtual_alias)s""") % context
-        )
-        self.append(textwrap.dedent("""
+                   -e 'N; /^\s*\\n\s*$/d; P; D' %(virtual_alias)s
            # Non-existent list archives produce exit code 1
            exit_code=0
            rmlist -a %(name)s || exit_code=$?
@ -175,12 +180,14 @@ class MailmanBackend(MailmanVirtualDomainBackend):
    def commit(self):
        context = self.get_context_files()
        self.append(textwrap.dedent("""
+            # Apply changes if needed
            if [[ $UPDATED_VIRTUAL_ALIAS == 1 ]]; then
                postmap %(virtual_alias)s
            fi
            if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then
                service postfix reload
-            fi""") % context
+            fi
+            exit $exit_code""") % context
        )
    
    def get_context_files(self):
--- a/orchestra/contrib/mailboxes/backends.py
+++ b/orchestra/contrib/mailboxes/backends.py
@ -364,7 +364,7 @@ class PostfixAddressBackend(PostfixAddressVirtualDomainBackend):
    
    def commit(self):
        context = self.get_context_files()
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
            # Apply changes if needed
            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && {
                service postfix reload
--- a/orchestra/contrib/orchestration/backends.py
+++ b/orchestra/contrib/orchestration/backends.py
@ -1,3 +1,4 @@
+import textwrap
 from functools import partial

 from django.apps import apps
@ -207,10 +208,10 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount):
        hook for executing something at the beging
        define functions or initialize state
        """
-        self.append(
-            'set -e\n'
-            'set -o pipefail\n'
-            'exit_code=0;\n'
+        self.append(textwrap.dedent("""\
+            set -e
+            set -o pipefail
+            exit_code=0""")
        )
    
    def commit(self):
--- a/orchestra/contrib/systemusers/backends.py
+++ b/orchestra/contrib/systemusers/backends.py
@ -31,27 +31,26 @@ class UNIXUserBackend(ServiceController):
        context['groups_arg'] = '--groups %s' % groups if groups else ''
        # TODO userd add will fail if %(user)s group already exists
        self.append(textwrap.dedent("""
-            # Update/create %(user)s user state
+            # Update/create user state for %(user)s
            if [[ $( id %(user)s ) ]]; then
                usermod %(user)s --home %(home)s \\
                    --password '%(password)s' \\
                    --shell %(shell)s %(groups_arg)s
            else
+                useradd_code=0
                useradd %(user)s --home %(home)s \\
                    --password '%(password)s' \\
-                    --shell %(shell)s %(groups_arg)s || {
-                        useradd_code=$?
-                        # User is logged in, kill and retry
-                        if [[ $useradd_code -eq 8 ]]; then
-                            pkill -u %(user)s; sleep 2
-                            pkill -9 -u %(user)s; sleep 1
-                            useradd %(user)s --home %(home)s \\
-                                --password '%(password)s' \\
-                                --shell %(shell)s %(groups_arg)s
-                        else
-                            exit $useradd_code
-                        fi
-                    }
+                    --shell %(shell)s %(groups_arg)s || useradd_code=$?
+                if [[ $useradd_code -eq 8 ]]; then
+                    # User is logged in, kill and retry
+                    pkill -u %(user)s; sleep 2
+                    pkill -9 -u %(user)s; sleep 1
+                    useradd %(user)s --home %(home)s \\
+                        --password '%(password)s' \\
+                        --shell %(shell)s %(groups_arg)s
+                elif [[ $useradd_code -ne 0 ]]; then
+                    exit $useradd_code
+                fi
            fi
            mkdir -p %(base_home)s
            chmod 750 %(base_home)s
@ -59,7 +58,7 @@ class UNIXUserBackend(ServiceController):
        )
        if context['home'] != context['base_home']:
            self.append(textwrap.dedent("""
-                # Set extra permissions since %(user)s home is inside %(mainuser)s home
+                # Set extra permissions: %(user)s home is inside %(mainuser)s home
                if [[ $(mount | grep "^$(df %(home)s|grep '^/')\s" | grep acl) ]]; then
                    # Accountn group as the owner
                    chown %(mainuser)s:%(mainuser)s %(home)s
@ -90,7 +89,7 @@ class UNIXUserBackend(ServiceController):
            nohup bash -c 'sleep 2 && killall -u %(user)s -s KILL' &> /dev/null &
            killall -u %(user)s || true
            userdel %(user)s || exit_code=$?
-            groupdel %(group)s || exit_code=$?
+            groupdel %(group)s || exit_code=$?\
            """) % context
        )
        if context['deleted_home']:
@ -132,14 +131,14 @@ class UNIXUserBackend(ServiceController):
                self.append(textwrap.dedent("""\
                    # Grant access to main user
                    find '%(perm_to)s' -type d %(exclude_acl)s \\
-                        -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;
+                        -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
                    """) % context
                )
        elif user.set_perm_action == 'revoke':
            self.append(textwrap.dedent("""\
                # Revoke permissions
                find '%(perm_to)s' %(exclude_acl)s \\
-                    -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;
+                    -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\
                """) % context
            )
        else:
@ -149,11 +148,10 @@ class UNIXUserBackend(ServiceController):
        context = {
            'path': user.path_to_validate,
        }
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
            if [[ ! -e '%(path)s' ]]; then
                echo "%(path)s path does not exists." >&2
-            fi
-            """) % context
+            fi""") % context
        )
    
    def get_groups(self, user):
--- a/orchestra/contrib/webapps/backends/init.py
+++ b/orchestra/contrib/webapps/backends/init.py
@ -15,17 +15,19 @@ class WebAppServiceMixin(object):
    )
    
    def create_webapp_dir(self, context):
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
+            # Create webapp dir
            CREATED=0
            [[ ! -e %(app_path)s ]] && CREATED=1
            mkdir -p %(app_path)s
-            chown %(user)s:%(group)s %(app_path)s
+            chown %(user)s:%(group)s %(app_path)s\
            """) % context
        )
    
    def set_under_construction(self, context):
        if context['under_construction_path']:
-            self.append(textwrap.dedent("""\
+            self.append(textwrap.dedent("""
+                # Set under construction if needed
                if [[ $CREATED == 1 && ! $(ls -A %(app_path)s) ]]; then
                    # Async wait 2 more seconds for other backends to lock app_path or cp under construction
                    nohup bash -c '
--- a/orchestra/contrib/webapps/backends/php.py
+++ b/orchestra/contrib/webapps/backends/php.py
@ -44,6 +44,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
    
    def save_fpm(self, webapp, context):
        self.append(textwrap.dedent("""
+            # Generate FPM configuration
            read -r -d '' fpm_config << 'EOF' || true
            %(fpm_config)s
            EOF
@ -58,7 +59,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
    
    def save_fcgid(self, webapp, context):
        self.append("mkdir -p %(wrapper_dir)s" % context)
-        self.append(textwrap.dedent("""\
+        self.append(textwrap.dedent("""
+            # Generate FCGID configuration
            read -r -d '' wrapper << 'EOF' || true
            %(wrapper)s
            EOF
@ -78,6 +80,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
        self.append("chown -R %(user)s:%(group)s %(wrapper_dir)s" % context)
        if context['cmd_options']:
            self.append(textwrap.dedent("""\
+                # FCGID options
                read -r -d '' cmd_options << 'EOF' || true
                %(cmd_options)s
                EOF
@ -122,6 +125,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
    
    def commit(self):
        self.append(textwrap.dedent("""
+            # Apply changes if needed
            if [[ $UPDATED_FPM -eq 1 ]]; then
                service php5-fpm reload
            fi
--- a/orchestra/contrib/webapps/backends/wordpress.py
+++ b/orchestra/contrib/webapps/backends/wordpress.py
@ -42,6 +42,7 @@ class WordPressBackend(WebAppServiceMixin, ServiceController):
            if (count(glob("%(app_path)s/*")) > 1) {
                die("App directory not empty.");
            }
+            // Download and untar wordpress (with caching system)
            shell_exec("mkdir -p %(app_path)s
                # Prevent other backends from writting here
                touch %(app_path)s/.lock
@ -67,6 +68,7 @@ class WordPressBackend(WebAppServiceMixin, ServiceController):
            }
            array_pop($secret_keys);
            
+            // setup wordpress database and keys config
            $config_file = str_replace('database_name_here', "%(db_name)s", $config_file);
            $config_file = str_replace('username_here', "%(db_user)s", $config_file);
            $config_file = str_replace('password_here', "%(password)s", $config_file);
@ -90,6 +92,8 @@ class WordPressBackend(WebAppServiceMixin, ServiceController):
            }
            exc('chown -R %(user)s:%(group)s %(app_path)s');
            
+            // Execute wordpress installation process
+            
            define('WP_CONTENT_DIR', 'wp-content/');
            define('WP_LANG_DIR', WP_CONTENT_DIR . '/languages' );
            define('WP_USE_THEMES', true);
--- a/orchestra/contrib/websites/backends/apache.py
+++ b/orchestra/contrib/websites/backends/apache.py
@ -99,7 +99,7 @@ class Apache2Backend(ServiceController):
                apache_conf += self.render_redirect_https(context)
            context['apache_conf'] = apache_conf.strip()
            self.append(textwrap.dedent("""
-                # Generate %(site_name)s Apache site config
+                # Generate Apache site config for %(site_name)s
                read -r -d '' apache_conf << 'EOF' || true
                %(apache_conf)s
                EOF
@ -112,7 +112,7 @@ class Apache2Backend(ServiceController):
            )
        if context['server_name'] and site.active:
            self.append(textwrap.dedent("""\
-                # Enable %(site_name)s site
+                # Enable site %(site_name)s
                if [[ ! -f %(sites_enabled)s ]]; then
                    a2ensite %(site_unique_name)s.conf
                    UPDATED_APACHE=1
@ -120,7 +120,7 @@ class Apache2Backend(ServiceController):
            )
        else:
            self.append(textwrap.dedent("""\
-                # Disable %(site_name)s site
+                # Disable site %(site_name)s
                if [[ -f %(sites_enabled)s ]]; then
                    a2dissite %(site_unique_name)s.conf;
                    UPDATED_APACHE=1
@ -130,7 +130,7 @@ class Apache2Backend(ServiceController):
    def delete(self, site):
        context = self.get_context(site)
        self.append(textwrap.dedent("""
-            # Remove %(site_name)s site configuration
+            # Remove site configuration for %(site_name)s
            a2dissite %(site_unique_name)s.conf && UPDATED_APACHE=1
            rm -f %(sites_available)s\
            """) % context