From 43eb744f819663736735a5c9922a99e60656ad7c Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 21 May 2015 17:53:59 +0000 Subject: [PATCH] More robust bash backends using heredoc --- TODO.md | 2 + orchestra/contrib/databases/backends.py | 41 +++++++++++------- orchestra/contrib/domains/backends.py | 27 ++++++++---- orchestra/contrib/lists/backends.py | 43 +++++++++++-------- orchestra/contrib/mailboxes/backends.py | 2 +- orchestra/contrib/orchestration/backends.py | 9 ++-- orchestra/contrib/systemusers/backends.py | 40 ++++++++--------- .../contrib/webapps/backends/__init__.py | 8 ++-- orchestra/contrib/webapps/backends/php.py | 6 ++- .../contrib/webapps/backends/wordpress.py | 4 ++ orchestra/contrib/websites/backends/apache.py | 8 ++-- 11 files changed, 113 insertions(+), 77 deletions(-) diff --git a/TODO.md b/TODO.md index f31ef7ee..12e80b51 100644 --- a/TODO.md +++ b/TODO.md @@ -382,3 +382,5 @@ http://wiki2.dovecot.org/Pigeonhole/Sieve/Examples # mail system users group? which one is more convinient? if main group does not exists, backend will fail! + +Bash/Python/PHPBackend diff --git a/orchestra/contrib/databases/backends.py b/orchestra/contrib/databases/backends.py index 204b4b8f..b1d49a3c 100644 --- a/orchestra/contrib/databases/backends.py +++ b/orchestra/contrib/databases/backends.py @@ -23,20 +23,21 @@ class MySQLBackend(ServiceController): if database.type != database.MYSQL: return context = self.get_context(database) - self.append( - "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context - ) # Not available on delete() context['owner'] = database.owner - # clean previous privileges - self.append("""mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'""" % context) + self.append(textwrap.dedent(""" + # Create database and re-set permissions + mysql -e 'CREATE DATABASE `%(database)s`;' || true + mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\ + """) % context + ) for user in database.users.all(): context.update({ 'username': user.username, 'grant': 'WITH GRANT OPTION' if user == context['owner'] else '' }) self.append(textwrap.dedent("""\ - mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;' \ + mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;'\ """) % context ) @@ -44,11 +45,19 @@ class MySQLBackend(ServiceController): if database.type != database.MYSQL: return context = self.get_context(database) - self.append("mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=$?" % context) - self.append("mysql mysql -e 'DELETE FROM db WHERE db = \"%(database)s\";'" % context) - + self.append(textwrap.dedent(""" + # Remove database %(database)s + mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=$? + mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\ + """) % context + ) + def commit(self): - self.append("mysql -e 'FLUSH PRIVILEGES;'") + self.append(textwrap.dedent(""" + # Apply permissions + mysql -e 'FLUSH PRIVILEGES;'\ + """) + ) super(MySQLBackend, self).commit() def get_context(self, database): @@ -75,11 +84,9 @@ class MySQLUserBackend(ServiceController): return context = self.get_context(user) self.append(textwrap.dedent("""\ - mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \ - """) % context - ) - self.append(textwrap.dedent("""\ - mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \ + # Create user %(username)s + mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true + mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";'\ """) % context ) @@ -87,12 +94,14 @@ class MySQLUserBackend(ServiceController): if user.type != user.MYSQL: return context = self.get_context(user) - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Delete user %(username)s mysql -e 'DROP USER "%(username)s"@"%(host)s";' || exit_code=$? \ """) % context ) def commit(self): + self.append("# Apply permissions") self.append("mysql -e 'FLUSH PRIVILEGES;'") def get_context(self, user): diff --git a/orchestra/contrib/domains/backends.py b/orchestra/contrib/domains/backends.py index 4cac7458..e5d23466 100644 --- a/orchestra/contrib/domains/backends.py +++ b/orchestra/contrib/domains/backends.py @@ -45,19 +45,21 @@ class Bind9MasterDomainBackend(ServiceController): def update_zone(self, domain, context): context['zone'] = ';; %(banner)s\n' % context context['zone'] += domain.render_zone() - self.append(textwrap.dedent(""" + self.append(textwrap.dedent("""\ + # Generate %(name)s zone file cat << 'EOF' > %(zone_path)s.tmp %(zone)s EOF diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1 # Because bind reload will not display any fucking error named-checkzone -k fail -n fail %(name)s %(zone_path)s.tmp - mv %(zone_path)s.tmp %(zone_path)s + mv %(zone_path)s.tmp %(zone_path)s\ """) % context ) def update_conf(self, context): self.append(textwrap.dedent(""" + # Update bind config file for %(name)s read -r -d '' conf << 'EOF' || true %(conf)s EOF @@ -68,8 +70,8 @@ class Bind9MasterDomainBackend(ServiceController): UPDATED=1 }""") % context ) - # Delete ex-top-domains that are now subdomains self.append(textwrap.dedent("""\ + # Delete ex-top-domains that are now subdomains sed -i -e '/zone\s\s*".*\.%(name)s".*/,/^\s*};\s*$/d' \\ -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s""") % context ) @@ -79,6 +81,7 @@ class Bind9MasterDomainBackend(ServiceController): def delete(self, domain): context = self.get_context(domain) + self.append('# Delete zone file for %(name)s' % context) self.append('rm -f %(zone_path)s;' % context) self.delete_conf(context) @@ -87,6 +90,7 @@ class Bind9MasterDomainBackend(ServiceController): # These can never be top level domains return self.append(textwrap.dedent(""" + # Delete config for %(name)s sed -e '/zone\s\s*"%(name)s".*/,/^\s*};\s*$/d' \\ -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s > %(conf_path)s.tmp""") % context ) @@ -95,7 +99,12 @@ class Bind9MasterDomainBackend(ServiceController): def commit(self): """ reload bind if needed """ - self.append('if [[ $UPDATED == 1 ]]; then service bind9 reload; fi') + self.append(textwrap.dedent(""" + # Apply changes + if [[ $UPDATED == 1 ]]; then + service bind9 reload + fi""") + ) def get_servers(self, domain, backend): """ Get related server IPs from registered backend routes """ @@ -180,12 +189,12 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): self.delete_conf(context) def commit(self): - """ ideally slave should be restarted after master """ - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Apply changes if [[ $UPDATED == 1 ]]; then + # Async restart, ideally after master nohup bash -c 'sleep 1 && service bind9 reload' &> /dev/null & - fi - """) + fi""") ) def get_context(self, domain): @@ -196,7 +205,7 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): 'masters': '; '.join(self.get_masters_ips(domain)) or 'none', 'conf_path': self.CONF_PATH, } - context['conf'] = textwrap.dedent(""" + context['conf'] = textwrap.dedent("""\ zone "%(name)s" { // %(banner)s type slave; diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index be2fd868..758d70b3 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -27,6 +27,7 @@ class MailmanVirtualDomainBackend(ServiceController): domain = context['address_domain'] if domain and self.is_local_domain(domain): self.append(textwrap.dedent(""" + # Add virtual domain %(address_domain)s [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || { echo '%(address_domain)s' >> %(virtual_alias_domains)s UPDATED_VIRTUAL_ALIAS_DOMAINS=1 @@ -39,7 +40,11 @@ class MailmanVirtualDomainBackend(ServiceController): def exclude_virtual_alias_domain(self, context): domain = context['address_domain'] if domain and self.is_last_domain(domain): - self.append("sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s" % context) + self.append(textwrap.dedent(""" + # Remove %(address_domain)s from virtual domains + sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s\ + """) % context + ) def save(self, mail_list): context = self.get_context(mail_list) @@ -52,10 +57,12 @@ class MailmanVirtualDomainBackend(ServiceController): def commit(self): context = self.get_context_files() self.append(textwrap.dedent(""" + # Apply changes if needed if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then service postfix reload fi""") % context ) + super(MailmanVirtualDomainBackend, self).commit() def get_context_files(self): return { @@ -108,15 +115,16 @@ class MailmanBackend(MailmanVirtualDomainBackend): def save(self, mail_list): context = self.get_context(mail_list) # Create list - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Create list %(name)s [[ ! -e '%(mailman_root)s/lists/%(name)s' ]] && { newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s' }""") % context) # Custom domain if mail_list.address: context['aliases'] = self.get_virtual_aliases(context) - # Preserve indentation self.append(textwrap.dedent("""\ + # Create list alias for custom domain aliases='%(aliases)s' if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then echo "${aliases}" >> %(virtual_alias)s @@ -128,27 +136,25 @@ class MailmanBackend(MailmanVirtualDomainBackend): echo "${aliases}" >> %(virtual_alias)s UPDATED_VIRTUAL_ALIAS=1 fi - fi""") % context - ) - self.append( - 'echo "require_explicit_destination = 0" | ' - '%(mailman_root)s/bin/config_list -i /dev/stdin %(name)s' % context - ) - self.append(textwrap.dedent("""\ - echo "host_name = '%(address_domain)s'" | \ + fi + echo "require_explicit_destination = 0" | \\ + %(mailman_root)s/bin/config_list -i /dev/stdin %(name)s + echo "host_name = '%(address_domain)s'" | \\ %(mailman_root)s/bin/config_list -i /dev/stdin %(name)s""") % context ) else: - # Cleanup shit self.append(textwrap.dedent("""\ + # Cleanup possible ex-custom domain if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s fi""") % context ) # Update if context['password'] is not None: - self.append( - '%(mailman_root)s/bin/change_pw --listname="%(name)s" --password="%(password)s"' % context + self.append(textwrap.dedent("""\ + # Re-set password + %(mailman_root)s/bin/change_pw --listname="%(name)s" --password="%(password)s"\ + """) % context ) self.include_virtual_alias_domain(context) if mail_list.active: @@ -160,10 +166,9 @@ class MailmanBackend(MailmanVirtualDomainBackend): context = self.get_context(mail_list) self.exclude_virtual_alias_domain(context) self.append(textwrap.dedent(""" + # Remove list %(name)s sed -i -e '/^.*\s%(name)s\(%(suffixes_regex)s\)\s*$/d' \\ - -e 'N; /^\s*\\n\s*$/d; P; D' %(virtual_alias)s""") % context - ) - self.append(textwrap.dedent(""" + -e 'N; /^\s*\\n\s*$/d; P; D' %(virtual_alias)s # Non-existent list archives produce exit code 1 exit_code=0 rmlist -a %(name)s || exit_code=$? @@ -175,12 +180,14 @@ class MailmanBackend(MailmanVirtualDomainBackend): def commit(self): context = self.get_context_files() self.append(textwrap.dedent(""" + # Apply changes if needed if [[ $UPDATED_VIRTUAL_ALIAS == 1 ]]; then postmap %(virtual_alias)s fi if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then service postfix reload - fi""") % context + fi + exit $exit_code""") % context ) def get_context_files(self): diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 50c2f13c..fe3bb983 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -364,7 +364,7 @@ class PostfixAddressBackend(PostfixAddressVirtualDomainBackend): def commit(self): context = self.get_context_files() - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" # Apply changes if needed [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py index 2de392ab..02d62422 100644 --- a/orchestra/contrib/orchestration/backends.py +++ b/orchestra/contrib/orchestration/backends.py @@ -1,3 +1,4 @@ +import textwrap from functools import partial from django.apps import apps @@ -207,10 +208,10 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): hook for executing something at the beging define functions or initialize state """ - self.append( - 'set -e\n' - 'set -o pipefail\n' - 'exit_code=0;\n' + self.append(textwrap.dedent("""\ + set -e + set -o pipefail + exit_code=0""") ) def commit(self): diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index 651e6da9..cbe3fe9f 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -31,27 +31,26 @@ class UNIXUserBackend(ServiceController): context['groups_arg'] = '--groups %s' % groups if groups else '' # TODO userd add will fail if %(user)s group already exists self.append(textwrap.dedent(""" - # Update/create %(user)s user state + # Update/create user state for %(user)s if [[ $( id %(user)s ) ]]; then usermod %(user)s --home %(home)s \\ --password '%(password)s' \\ --shell %(shell)s %(groups_arg)s else + useradd_code=0 useradd %(user)s --home %(home)s \\ --password '%(password)s' \\ - --shell %(shell)s %(groups_arg)s || { - useradd_code=$? - # User is logged in, kill and retry - if [[ $useradd_code -eq 8 ]]; then - pkill -u %(user)s; sleep 2 - pkill -9 -u %(user)s; sleep 1 - useradd %(user)s --home %(home)s \\ - --password '%(password)s' \\ - --shell %(shell)s %(groups_arg)s - else - exit $useradd_code - fi - } + --shell %(shell)s %(groups_arg)s || useradd_code=$? + if [[ $useradd_code -eq 8 ]]; then + # User is logged in, kill and retry + pkill -u %(user)s; sleep 2 + pkill -9 -u %(user)s; sleep 1 + useradd %(user)s --home %(home)s \\ + --password '%(password)s' \\ + --shell %(shell)s %(groups_arg)s + elif [[ $useradd_code -ne 0 ]]; then + exit $useradd_code + fi fi mkdir -p %(base_home)s chmod 750 %(base_home)s @@ -59,7 +58,7 @@ class UNIXUserBackend(ServiceController): ) if context['home'] != context['base_home']: self.append(textwrap.dedent(""" - # Set extra permissions since %(user)s home is inside %(mainuser)s home + # Set extra permissions: %(user)s home is inside %(mainuser)s home if [[ $(mount | grep "^$(df %(home)s|grep '^/')\s" | grep acl) ]]; then # Accountn group as the owner chown %(mainuser)s:%(mainuser)s %(home)s @@ -90,7 +89,7 @@ class UNIXUserBackend(ServiceController): nohup bash -c 'sleep 2 && killall -u %(user)s -s KILL' &> /dev/null & killall -u %(user)s || true userdel %(user)s || exit_code=$? - groupdel %(group)s || exit_code=$? + groupdel %(group)s || exit_code=$?\ """) % context ) if context['deleted_home']: @@ -132,14 +131,14 @@ class UNIXUserBackend(ServiceController): self.append(textwrap.dedent("""\ # Grant access to main user find '%(perm_to)s' -type d %(exclude_acl)s \\ - -exec setfacl -m d:u:%(mainuser)s:rwx {} \\; + -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\ """) % context ) elif user.set_perm_action == 'revoke': self.append(textwrap.dedent("""\ # Revoke permissions find '%(perm_to)s' %(exclude_acl)s \\ - -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\; + -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\ """) % context ) else: @@ -149,11 +148,10 @@ class UNIXUserBackend(ServiceController): context = { 'path': user.path_to_validate, } - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" if [[ ! -e '%(path)s' ]]; then echo "%(path)s path does not exists." >&2 - fi - """) % context + fi""") % context ) def get_groups(self, user): diff --git a/orchestra/contrib/webapps/backends/__init__.py b/orchestra/contrib/webapps/backends/__init__.py index d81a00d2..da79b48d 100644 --- a/orchestra/contrib/webapps/backends/__init__.py +++ b/orchestra/contrib/webapps/backends/__init__.py @@ -15,17 +15,19 @@ class WebAppServiceMixin(object): ) def create_webapp_dir(self, context): - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Create webapp dir CREATED=0 [[ ! -e %(app_path)s ]] && CREATED=1 mkdir -p %(app_path)s - chown %(user)s:%(group)s %(app_path)s + chown %(user)s:%(group)s %(app_path)s\ """) % context ) def set_under_construction(self, context): if context['under_construction_path']: - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Set under construction if needed if [[ $CREATED == 1 && ! $(ls -A %(app_path)s) ]]; then # Async wait 2 more seconds for other backends to lock app_path or cp under construction nohup bash -c ' diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py index b0521f72..39161e72 100644 --- a/orchestra/contrib/webapps/backends/php.py +++ b/orchestra/contrib/webapps/backends/php.py @@ -44,6 +44,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): def save_fpm(self, webapp, context): self.append(textwrap.dedent(""" + # Generate FPM configuration read -r -d '' fpm_config << 'EOF' || true %(fpm_config)s EOF @@ -58,7 +59,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController): def save_fcgid(self, webapp, context): self.append("mkdir -p %(wrapper_dir)s" % context) - self.append(textwrap.dedent("""\ + self.append(textwrap.dedent(""" + # Generate FCGID configuration read -r -d '' wrapper << 'EOF' || true %(wrapper)s EOF @@ -78,6 +80,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.append("chown -R %(user)s:%(group)s %(wrapper_dir)s" % context) if context['cmd_options']: self.append(textwrap.dedent("""\ + # FCGID options read -r -d '' cmd_options << 'EOF' || true %(cmd_options)s EOF @@ -122,6 +125,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): def commit(self): self.append(textwrap.dedent(""" + # Apply changes if needed if [[ $UPDATED_FPM -eq 1 ]]; then service php5-fpm reload fi diff --git a/orchestra/contrib/webapps/backends/wordpress.py b/orchestra/contrib/webapps/backends/wordpress.py index 5a2454ed..62e85740 100644 --- a/orchestra/contrib/webapps/backends/wordpress.py +++ b/orchestra/contrib/webapps/backends/wordpress.py @@ -42,6 +42,7 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): if (count(glob("%(app_path)s/*")) > 1) { die("App directory not empty."); } + // Download and untar wordpress (with caching system) shell_exec("mkdir -p %(app_path)s # Prevent other backends from writting here touch %(app_path)s/.lock @@ -67,6 +68,7 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): } array_pop($secret_keys); + // setup wordpress database and keys config $config_file = str_replace('database_name_here', "%(db_name)s", $config_file); $config_file = str_replace('username_here', "%(db_user)s", $config_file); $config_file = str_replace('password_here', "%(password)s", $config_file); @@ -90,6 +92,8 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): } exc('chown -R %(user)s:%(group)s %(app_path)s'); + // Execute wordpress installation process + define('WP_CONTENT_DIR', 'wp-content/'); define('WP_LANG_DIR', WP_CONTENT_DIR . '/languages' ); define('WP_USE_THEMES', true); diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py index b2161feb..c2b19fee 100644 --- a/orchestra/contrib/websites/backends/apache.py +++ b/orchestra/contrib/websites/backends/apache.py @@ -99,7 +99,7 @@ class Apache2Backend(ServiceController): apache_conf += self.render_redirect_https(context) context['apache_conf'] = apache_conf.strip() self.append(textwrap.dedent(""" - # Generate %(site_name)s Apache site config + # Generate Apache site config for %(site_name)s read -r -d '' apache_conf << 'EOF' || true %(apache_conf)s EOF @@ -112,7 +112,7 @@ class Apache2Backend(ServiceController): ) if context['server_name'] and site.active: self.append(textwrap.dedent("""\ - # Enable %(site_name)s site + # Enable site %(site_name)s if [[ ! -f %(sites_enabled)s ]]; then a2ensite %(site_unique_name)s.conf UPDATED_APACHE=1 @@ -120,7 +120,7 @@ class Apache2Backend(ServiceController): ) else: self.append(textwrap.dedent("""\ - # Disable %(site_name)s site + # Disable site %(site_name)s if [[ -f %(sites_enabled)s ]]; then a2dissite %(site_unique_name)s.conf; UPDATED_APACHE=1 @@ -130,7 +130,7 @@ class Apache2Backend(ServiceController): def delete(self, site): context = self.get_context(site) self.append(textwrap.dedent(""" - # Remove %(site_name)s site configuration + # Remove site configuration for %(site_name)s a2dissite %(site_unique_name)s.conf && UPDATED_APACHE=1 rm -f %(sites_available)s\ """) % context