django-orchestra-test/orchestra/contrib/mailboxes/backends.py

492 lines
20 KiB
Python
Raw Normal View History

2014-10-09 17:04:12 +00:00
import logging
import textwrap
2014-08-22 15:31:44 +00:00
2014-10-06 14:57:02 +00:00
from django.core.exceptions import ObjectDoesNotExist
2014-08-22 15:31:44 +00:00
from django.utils.translation import ugettext_lazy as _
2015-04-05 18:02:36 +00:00
from orchestra.contrib.orchestration import ServiceController, replace
2015-04-05 10:46:24 +00:00
from orchestra.contrib.resources import ServiceMonitor
#from orchestra.utils.humanize import unit_to_bytes
2014-08-22 15:31:44 +00:00
from . import settings
2014-09-26 15:05:20 +00:00
from .models import Address
2014-08-22 15:31:44 +00:00
2014-10-06 14:57:02 +00:00
# TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
# TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
# TODO mount the filesystem with "nosuid" option
2014-08-22 15:31:44 +00:00
2014-10-06 14:57:02 +00:00
2014-10-09 17:04:12 +00:00
logger = logging.getLogger(__name__)
2015-04-05 18:02:36 +00:00
class UNIXUserMaildirBackend(ServiceController):
2015-04-23 19:46:23 +00:00
"""
Assumes that all system users on this servers all mail accounts.
If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes
"""
2015-04-05 18:02:36 +00:00
verbose_name = _("UNIX maildir user")
2015-03-10 21:51:10 +00:00
model = 'mailboxes.Mailbox'
def save(self, mailbox):
context = self.get_context(mailbox)
self.append(textwrap.dedent("""
if [[ $( id %(user)s ) ]]; then
usermod %(user)s --password '%(password)s' --shell %(initial_shell)s
else
useradd %(user)s --home %(home)s --password '%(password)s'
fi
mkdir -p %(home)s
chmod 751 %(home)s
chown %(user)s:%(group)s %(home)s""") % context
)
if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'):
self.set_quota(mailbox, context)
def set_quota(self, mailbox, context):
context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale()
#unit_to_bytes(mailbox.resources.disk.unit)
2015-03-10 21:51:10 +00:00
self.append(textwrap.dedent("""
mkdir -p %(home)s/Maildir
chown %(user)s:%(group)s %(home)s/Maildir
if [[ ! -f %(home)s/Maildir/maildirsize ]]; then
echo "%(quota)iS" > %(home)s/Maildir/maildirsize
chown %(user)s:%(group)s %(home)s/Maildir/maildirsize
else
sed -i '1s/.*/%(quota)iS/' %(home)s/Maildir/maildirsize
fi""") % context
)
def delete(self, mailbox):
context = self.get_context(mailbox)
2015-04-23 14:34:04 +00:00
self.append('mv %(home)s %(home)s.deleted || exit_code=1' % context)
2015-03-10 21:51:10 +00:00
self.append(textwrap.dedent("""
{ sleep 2 && killall -u %(user)s -s KILL; } &
killall -u %(user)s || true
userdel %(user)s || true
groupdel %(user)s || true""") % context
)
def get_context(self, mailbox):
context = {
'user': mailbox.name,
'group': mailbox.name,
'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
'home': mailbox.get_home(),
'initial_shell': '/dev/null',
}
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2015-03-10 21:51:10 +00:00
2015-04-05 18:02:36 +00:00
class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
2015-04-23 19:46:23 +00:00
"""
WARNING: This backends is not fully implemented
"""
2015-04-05 18:02:36 +00:00
verbose_name = _("Dovecot-Postfix virtualuser")
2014-10-17 10:04:47 +00:00
model = 'mailboxes.Mailbox'
2014-09-30 14:46:29 +00:00
# TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data
2014-08-22 15:31:44 +00:00
DEFAULT_GROUP = 'postfix'
2014-10-06 14:57:02 +00:00
def set_user(self, context):
self.append(textwrap.dedent("""
2015-03-10 21:51:10 +00:00
if [[ $( grep "^%(user)s:" %(passwd_path)s ) ]]; then
sed -i 's#^%(user)s:.*#%(passwd)s#' %(passwd_path)s
else
2014-10-06 14:57:02 +00:00
echo '%(passwd)s' >> %(passwd_path)s
fi""") % context
)
2014-08-22 15:31:44 +00:00
self.append("mkdir -p %(home)s" % context)
2015-02-24 09:34:26 +00:00
self.append("chown %(uid)s:%(gid)s %(home)s" % context)
2014-08-22 15:31:44 +00:00
2014-10-09 17:04:12 +00:00
def set_mailbox(self, context):
self.append(textwrap.dedent("""
2015-03-10 21:51:10 +00:00
if [[ ! $(grep "^%(user)s@%(mailbox_domain)s\s" %(virtual_mailbox_maps)s) ]]; then
echo "%(user)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s
2014-10-09 17:04:12 +00:00
UPDATED_VIRTUAL_MAILBOX_MAPS=1
fi""") % context
)
2014-10-09 17:04:12 +00:00
2014-08-22 15:31:44 +00:00
def generate_filter(self, mailbox, context):
2015-03-10 21:51:10 +00:00
self.append("doveadm mailbox create -u %(user)s Spam" % context)
2014-10-27 13:29:02 +00:00
context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context
2014-10-09 17:04:12 +00:00
filtering = mailbox.get_filtering()
if filtering:
context['filtering'] = '# %(banner)s\n' + filtering
2014-10-28 09:51:27 +00:00
self.append("mkdir -p $(dirname '%(filtering_path)s')" % context)
2014-10-09 17:04:12 +00:00
self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
2014-08-22 15:31:44 +00:00
else:
2014-10-09 17:04:12 +00:00
self.append("rm -f %(filtering_path)s" % context)
2014-08-22 15:31:44 +00:00
def save(self, mailbox):
context = self.get_context(mailbox)
2014-10-06 14:57:02 +00:00
self.set_user(context)
2014-10-09 17:04:12 +00:00
self.set_mailbox(context)
2014-08-22 15:31:44 +00:00
self.generate_filter(mailbox, context)
def delete(self, mailbox):
context = self.get_context(mailbox)
2015-04-09 14:32:10 +00:00
self.append(textwrap.dedent("""\
{ sleep 2 && killall -u %(uid)s -s KILL; } &
killall -u %(uid)s || true
sed -i '/^%(user)s:.*/d' %(passwd_path)s
sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s
UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context
)
if context['deleted_home']:
self.append("mv %(home)s %(deleted_home)s || exit_code=1" % context)
else:
self.append("rm -fr %(home)s" % context)
2014-08-22 15:31:44 +00:00
2014-10-06 14:57:02 +00:00
def get_extra_fields(self, mailbox, context):
context['quota'] = self.get_quota(mailbox)
return 'userdb_mail=maildir:~/Maildir {quota}'.format(**context)
def get_quota(self, mailbox):
try:
quota = mailbox.resources.disk.allocated
except (AttributeError, ObjectDoesNotExist):
return ''
unit = mailbox.resources.disk.unit[0].upper()
return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit)
2014-10-09 17:04:12 +00:00
def commit(self):
context = {
2014-10-17 10:04:47 +00:00
'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH
2014-10-09 17:04:12 +00:00
}
2014-11-27 19:17:26 +00:00
self.append(textwrap.dedent("""\
[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && {
postmap %(virtual_mailbox_maps)s
}""") % context
)
2014-10-09 17:04:12 +00:00
2014-08-22 15:31:44 +00:00
def get_context(self, mailbox):
context = {
2014-09-30 14:46:29 +00:00
'name': mailbox.name,
2015-03-10 21:51:10 +00:00
'user': mailbox.name,
2014-10-06 14:57:02 +00:00
'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
'uid': 10000 + mailbox.pk,
'gid': 10000 + mailbox.pk,
'group': self.DEFAULT_GROUP,
'quota': self.get_quota(mailbox),
2014-10-17 10:04:47 +00:00
'passwd_path': settings.MAILBOXES_PASSWD_PATH,
2015-04-09 14:32:10 +00:00
'home': mailbox.get_home(),
2014-10-09 17:04:12 +00:00
'banner': self.get_banner(),
2014-10-17 10:04:47 +00:00
'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH,
'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
2014-08-22 15:31:44 +00:00
}
2014-10-06 14:57:02 +00:00
context['extra_fields'] = self.get_extra_fields(mailbox, context)
2015-04-09 14:32:10 +00:00
context.update({
'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context),
'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context,
})
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2014-08-22 15:31:44 +00:00
class PostfixAddressBackend(ServiceController):
2015-04-23 19:46:23 +00:00
"""
Addresses based on Postfix virtual alias domains.
<tt>MAILBOXES_LOCAL_DOMAIN = '%s'
MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH = '%s'
MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH = '%s'</tt>
"""
format_docstring = (
settings.MAILBOXES_LOCAL_DOMAIN,
settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH,
settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH,
)
2014-08-22 15:31:44 +00:00
verbose_name = _("Postfix address")
2014-10-17 10:04:47 +00:00
model = 'mailboxes.Address'
related_models = (
('mailboxes.Mailbox', 'addresses'),
)
2014-08-22 15:31:44 +00:00
2014-10-09 17:04:12 +00:00
def include_virtual_alias_domain(self, context):
2015-04-09 14:32:10 +00:00
if context['domain'] != context['local_domain']:
self.append(textwrap.dedent("""
[[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || {
echo '%(domain)s' >> %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
}""") % context
)
2014-08-22 15:31:44 +00:00
2014-10-09 17:04:12 +00:00
def exclude_virtual_alias_domain(self, context):
2014-08-22 15:31:44 +00:00
domain = context['domain']
if not Address.objects.filter(domain=domain).exists():
2015-04-05 18:02:36 +00:00
self.append("sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s" % context)
2014-10-09 17:04:12 +00:00
def update_virtual_alias_maps(self, address, context):
# Virtual mailbox stuff
# destination = []
# for mailbox in address.get_mailboxes():
# context['mailbox'] = mailbox
2015-04-09 14:32:10 +00:00
# destination.append("%(mailbox)s@%(local_domain)s" % context)
# for forward in address.forward:
# if '@' in forward:
# destination.append(forward)
destination = address.destination
2014-10-09 17:04:12 +00:00
if destination:
context['destination'] = destination
2014-10-09 17:04:12 +00:00
self.append(textwrap.dedent("""
2015-04-05 18:02:36 +00:00
LINE='%(email)s\t%(destination)s'
if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
2014-10-09 17:04:12 +00:00
echo "${LINE}" >> %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
else
if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then
sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
fi
2014-11-27 19:17:26 +00:00
fi""") % context)
2014-10-09 17:04:12 +00:00
else:
logger.warning("Address %i is empty" % address.pk)
2015-04-05 18:02:36 +00:00
self.append("sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s" % context)
2014-10-09 17:04:12 +00:00
self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1')
2014-08-22 15:31:44 +00:00
2014-10-09 17:04:12 +00:00
def exclude_virtual_alias_maps(self, context):
2014-10-06 14:57:02 +00:00
self.append(textwrap.dedent("""
2015-04-05 18:02:36 +00:00
if [[ $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then
sed -i '/^%(email)s\s.*$/d' %(virtual_alias_maps)s
2014-10-09 17:04:12 +00:00
UPDATED_VIRTUAL_ALIAS_MAPS=1
2014-11-27 19:17:26 +00:00
fi""") % context)
2014-08-22 15:31:44 +00:00
def save(self, address):
context = self.get_context(address)
2014-10-09 17:04:12 +00:00
self.include_virtual_alias_domain(context)
self.update_virtual_alias_maps(address, context)
2014-08-22 15:31:44 +00:00
def delete(self, address):
context = self.get_context(address)
2014-10-09 17:04:12 +00:00
self.exclude_virtual_alias_domain(context)
self.exclude_virtual_alias_maps(context)
2014-08-22 15:31:44 +00:00
def commit(self):
context = self.get_context_files()
2014-10-06 14:57:02 +00:00
self.append(textwrap.dedent("""
2014-10-09 17:04:12 +00:00
[[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; }
2015-04-09 14:32:10 +00:00
[[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload; }
2014-11-27 19:17:26 +00:00
""") % context
)
self.append('exit 0')
2014-08-22 15:31:44 +00:00
def get_context_files(self):
return {
2014-10-17 10:04:47 +00:00
'virtual_alias_domains': settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH,
'virtual_alias_maps': settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH
2014-08-22 15:31:44 +00:00
}
def get_context(self, address):
context = self.get_context_files()
context.update({
'domain': address.domain,
'email': address.email,
2015-04-09 14:32:10 +00:00
'local_domain': settings.MAILBOXES_LOCAL_DOMAIN,
2014-08-22 15:31:44 +00:00
})
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2014-08-22 15:31:44 +00:00
class AutoresponseBackend(ServiceController):
2015-04-23 19:46:23 +00:00
"""
WARNING: not implemented
"""
2014-08-22 15:31:44 +00:00
verbose_name = _("Mail autoresponse")
2015-04-05 18:02:36 +00:00
model = 'mailboxes.Autoresponse'
2014-08-22 15:31:44 +00:00
2015-04-05 18:02:36 +00:00
class DovecotMaildirDisk(ServiceMonitor):
2014-11-18 13:59:21 +00:00
"""
Maildir disk usage based on Dovecot maildirsize file
http://wiki2.dovecot.org/Quota/Maildir
2015-04-23 19:46:23 +00:00
MAILBOXES_MAILDIRSIZE_PATH = '%s'
2014-11-18 13:59:21 +00:00
"""
2015-04-23 19:46:23 +00:00
format_docstring = (
settings.MAILBOXES_MAILDIRSIZE_PATH,
)
2014-10-17 10:04:47 +00:00
model = 'mailboxes.Mailbox'
2014-08-22 15:31:44 +00:00
resource = ServiceMonitor.DISK
2015-04-05 18:02:36 +00:00
verbose_name = _("Dovecot Maildir size")
2014-08-22 15:31:44 +00:00
2014-11-18 13:59:21 +00:00
def prepare(self):
2015-04-05 18:02:36 +00:00
super(DovecotMaildirDisk, self).prepare()
2014-11-18 13:59:21 +00:00
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z")
self.append(textwrap.dedent("""\
function monitor () {
2015-03-23 15:36:51 +00:00
awk 'BEGIN { size = 0 } NR > 1 { size += $1 } END { print size }' $1 || echo 0
2014-11-18 13:59:21 +00:00
}"""))
2014-08-22 15:31:44 +00:00
def monitor(self, mailbox):
context = self.get_context(mailbox)
2014-11-18 13:59:21 +00:00
self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context)
2014-08-22 15:31:44 +00:00
def get_context(self, mailbox):
2014-10-23 21:25:44 +00:00
context = {
2014-11-17 14:17:33 +00:00
'home': mailbox.get_home(),
2014-10-06 14:57:02 +00:00
'object_id': mailbox.pk
2014-10-23 21:25:44 +00:00
}
2014-11-17 14:17:33 +00:00
context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')
2015-04-05 18:02:36 +00:00
class PostfixMailscannerTraffic(ServiceMonitor):
"""
2015-04-23 19:46:23 +00:00
A high-performance log parser.
Reads the mail.log file only once, for all users.
MAILBOXES_MAIL_LOG_PATH = '%s'
"""
2015-04-23 19:46:23 +00:00
format_docstring = (
settings.MAILBOXES_MAIL_LOG_PATH,
)
model = 'mailboxes.Mailbox'
resource = ServiceMonitor.TRAFFIC
2015-04-05 18:02:36 +00:00
verbose_name = _("Postfix-Mailscanner traffic")
script_executable = '/usr/bin/python'
def prepare(self):
2015-04-05 18:02:36 +00:00
mail_log = settings.MAILBOXES_MAIL_LOG_PATH
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'mail_logs': str((mail_log, mail_log+'.1')),
}
self.append(textwrap.dedent("""\
import re
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
# Converts orchestra's UTC dates to local timezone
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
maillogs = {mail_logs}
end_datetime = to_local_timezone('{current_date}')
end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
months = {{
"Jan": "01",
"Feb": "02",
"Mar": "03",
"Apr": "04",
"May": "05",
"Jun": "06",
"Jul": "07",
"Aug": "08",
"Sep": "09",
"Oct": "10",
"Nov": "11",
"Dec": "12",
}}
def inside_period(month, day, time, ini_date):
global months
global end_datetime
2015-04-09 14:32:10 +00:00
# Mar 9 17:13:22
month = months[month]
year = end_datetime.year
if month == '12' and end_datetime.month == 1:
year = year+1
2015-04-09 14:32:10 +00:00
if len(day) == 1:
day = '0' + day
date = str(year) + month + day
date += time.replace(':', '')
return ini_date < int(date) < end_date
users = {{}}
delivers = {{}}
reverse = {{}}
def prepare(object_id, mailbox, ini_date):
global users
global delivers
global reverse
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[mailbox] = (ini_date, object_id)
delivers[mailbox] = set()
reverse[mailbox] = set()
def monitor(users, delivers, reverse, maillogs):
targets = {{}}
counter = {{}}
user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
for maillog in maillogs:
try:
with open(maillog, 'r') as maillog:
for line in maillog.readlines():
# Only search for Authenticated sendings
if '(Authenticated sender: ' in line:
username = user_regex.search(line).groups()[0]
try:
sender = users[username]
except KeyError:
continue
else:
month, day, time, __, proc, id = line.split()[:6]
if inside_period(month, day, time, sender[0]):
# Add new email
delivers[id[:-1]] = username
# Look for a MailScanner requeue ID
elif ' Requeue: ' in line:
id, __, req_id = line.split()[6:9]
id = id.split('.')[0]
try:
username = delivers[id]
except KeyError:
pass
else:
2015-04-16 13:15:21 +00:00
targets[req_id] = (username, 0)
reverse[username].add(req_id)
# Look for the mail size and count the number of recipients of each email
else:
try:
month, day, time, __, proc, req_id, __, msize = line.split()[:8]
except ValueError:
# not interested in this line
continue
if proc.startswith('postfix/'):
req_id = req_id[:-1]
if msize.startswith('size='):
try:
target = targets[req_id]
except KeyError:
pass
else:
targets[req_id] = (target[0], int(msize[5:-1]))
elif proc.startswith('postfix/smtp'):
try:
target = targets[req_id]
except KeyError:
pass
else:
if inside_period(month, day, time, users[target[0]][0]):
try:
counter[req_id] += 1
except KeyError:
counter[req_id] = 1
except IOError as e:
sys.stderr.write(e)
2015-04-16 13:15:21 +00:00
for username, opts in users.iteritems():
size = 0
for req_id in reverse[username]:
size += targets[req_id][1] * counter.get(req_id, 0)
print opts[1], size
""").format(**context)
)
def commit(self):
self.append('monitor(users, delivers, reverse, maillogs)')
def monitor(self, mailbox):
context = self.get_context(mailbox)
self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context)
def get_context(self, mailbox):
2015-04-05 18:02:36 +00:00
context = {
'mailbox': mailbox.name,
'object_id': mailbox.pk,
'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
2015-04-05 18:02:36 +00:00
return replace(context, "'", '"')