pangea
/
django-orchestra
7
0
Fork 0
Code Issues 1 Pull Requests 2 Packages Projects Releases Wiki Activity

Compare commits

base: pangea:1724309769cdf5336a24f4d40254723cee3a3c7e
Branches Tags
pangea:main
pangea:musician
pangea:django-3.2-compat
pangea:sso
pangea:master_old
pangea:master
pangea:oauth2
pangea:trustchain
..
compare: pangea:3063e78dd921b84f22e0e4bee234d071f034eb5e
Branches Tags
pangea:main
pangea:musician
pangea:django-3.2-compat
pangea:sso
pangea:master_old
pangea:master
pangea:oauth2
pangea:trustchain

No commits in common. "1724309769cdf5336a24f4d40254723cee3a3c7e" and "3063e78dd921b84f22e0e4bee234d071f034eb5e" have entirely different histories.
1724309769 ... 3063e78dd9

2 changed files with 67 additions and 90 deletions
Show Stats Expand all files Collapse all files

154
orchestra/contrib/mailboxes/backends.py
View File

@ -494,7 +494,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
model = 'mailboxes.Mailbox' model = 'mailboxes.Mailbox'
resource = ServiceMonitor.TRAFFIC resource = ServiceMonitor.TRAFFIC
verbose_name = _("Postfix-Mailscanner traffic") verbose_name = _("Postfix-Mailscanner traffic")
script_executable = '/usr/bin/python3' script_executable = '/usr/bin/python'
monthly_sum_old_values = True monthly_sum_old_values = True
doc_settings = (settings, doc_settings = (settings,
('MAILBOXES_MAIL_LOG_PATH',) ('MAILBOXES_MAIL_LOG_PATH',)
@ -524,10 +524,6 @@ class PostfixMailscannerTraffic(ServiceMonitor):
end_date = int(end_datetime.strftime('%Y%m%d%H%M%S')) end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
months = dict((m, '%02d' % n) for n, m in enumerate(months, 1)) months = dict((m, '%02d' % n) for n, m in enumerate(months, 1))
users = {{}}
sends = {{}}
register_imap_traffic = False
register_pop_traffic = False
def inside_period(month, day, time, ini_date): def inside_period(month, day, time, ini_date):
global months global months
@ -542,110 +538,92 @@ class PostfixMailscannerTraffic(ServiceMonitor):
date = str(year) + month + day date = str(year) + month + day
date += time.replace(':', '') date += time.replace(':', '')
return ini_date < int(date) < end_date return ini_date < int(date) < end_date
def search_username(pattern, users, line): users = {{}}
match = pattern.search(line) delivers = {{}}
if not match: reverse = {{}}
return None
username = match.groups(1)[0]
if username not in users.keys():
return None
return username
def search_size(line, users, username, pattern):
month, day, time, req_id = line.split()[:4]
if inside_period(month, day, time, users[username][0]):
group = req_id.split('<')[-1][:-2]
matches = pattern.search(line)
if not matches:
return None, None
return group, matches
return None, None
def prepare(object_id, mailbox, ini_date): def prepare(object_id, mailbox, ini_date):
global users global users
global sends global delivers
global reverse
ini_date = to_local_timezone(ini_date) ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[mailbox] = (ini_date, object_id) users[mailbox] = (ini_date, object_id)
sends[mailbox] = {{}} delivers[mailbox] = set()
reverse[mailbox] = set()
def monitor(users, sends, maillogs): def monitor(users, delivers, reverse, maillogs):
grupos = [] targets = {{}}
sasl_username_pattern = re.compile(r'sasl_username=([a-zA-Z0-9\.\-_]+)') counter = {{}}
size_pattern = re.compile(r'size=(\d+),') user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
pop_username_pattern = re.compile(r' pop3\(([^)].*)\)')
pop_size_pattern = re.compile(r'size=(\d+)')
imap_username_pattern = re.compile(r' imap\(([^)].*)\)')
imap_size_pattern = re.compile(r"in=(\d+) out=(\d+)")
for maillog in maillogs: for maillog in maillogs:
try: try:
with open(maillog, 'r') as maillog: with open(maillog, 'r') as maillog:
for line in maillog.readlines(): for line in maillog.readlines():
# Only search for Authenticated sendings # Only search for Authenticated sendings
if 'sasl_username=' in line: if '(Authenticated sender: ' in line:
# si el usuario es uno de los elegidos y el rango de tiempo es correcto username = user_regex.search(line).groups()[0]
# recoge el id de grupo try:
username = search_username(sasl_username_pattern, users, line) sender = users[username]
if username is None: except KeyError:
continue continue
else:
month, day, time, __, __, req_id = line.split()[:6] month, day, time, __, proc, id = line.split()[:6]
if inside_period(month, day, time, users[username][0]): if inside_period(month, day, time, sender[0]):
group = req_id[:-1] # Add new email
sends[username][group] = 0 delivers[id[:-1]] = username
grupos.append(group) # Look for a MailScanner requeue ID
elif ' Requeue: ' in line:
id, __, req_id = line.split()[6:9]
id = id.split('.')[0]
try:
username = delivers[id]
except KeyError:
pass
else:
targets[req_id] = (username, 0)
reverse[username].add(req_id)
# Look for the mail size and count the number of recipients of each email
else: else:
# busca el size de envios donde se alla anadido el groupID anteriormente, try:
# una vez encontrado borra el groupID month, day, time, __, proc, req_id, __, msize = line.split()[:8]
for id in grupos: except ValueError:
if id in line: # not interested in this line
match = size_pattern.search(line) continue
if not match: if proc.startswith('postfix/'):
continue req_id = req_id[:-1]
for k, v in sends.items(): if msize.startswith('size='):
if id in sends[k].keys(): try:
sends[k][id] += int(match.groups(1)[0]) target = targets[req_id]
grupos.remove(id) except KeyError:
pass
# pop trafic else:
if register_pop_traffic: targets[req_id] = (target[0], int(msize[5:-1]))
if 'pop3(' in line and 'size' in line: elif proc.startswith('postfix/smtp'):
username = search_username(pop_username_pattern, users, line) try:
if username is None: target = targets[req_id]
continue except KeyError:
group, matches = search_size(line, users, username, pop_size_pattern) pass
if group is not None and matches is not None : else:
sends[username][group] = int(matches.groups(1)[0]) if inside_period(month, day, time, users[target[0]][0]):
try:
# imap trafic counter[req_id] += 1
if register_imap_traffic: except KeyError:
if 'imap(' in line and 'out=' in line: counter[req_id] = 1
username = search_username(imap_username_pattern, users, line)
if username is None:
continue
group, matches = search_size(line, users, username, imap_size_pattern)
if group is not None and matches is not None :
value = int(matches.group(1)) + int(matches.group(2))
sends[username][group] = value
except IOError as e: except IOError as e:
sys.stderr.write(str(e)+'\\n') sys.stderr.write(str(e)+'\\n')
# devolver la sumatoria de valores a orchestra (id_user, size) for username, opts in users.iteritems():
for username, opts in users.items(): size = 0
total_size = 0 for req_id in reverse[username]:
for size in sends[username].values(): size += targets[req_id][1] * counter.get(req_id, 0)
total_size += size print opts[1], size
print(f"{{opts[1]}} {{total_size}}")
""").format(**context) """).format(**context)
) )
def commit(self): def commit(self):
self.append('monitor(users, sends, maillogs)') self.append('monitor(users, delivers, reverse, maillogs)')
def monitor(self, mailbox): def monitor(self, mailbox):
context = self.get_context(mailbox) context = self.get_context(mailbox)

3
orchestra/contrib/resources/aggregations.py
View File

@ -34,8 +34,7 @@ class Last(Aggregation):
def filter(self, dataset, date=None): def filter(self, dataset, date=None):
lastdataset = dataset.order_by('-id').first() lastdataset = dataset.order_by('-id').first()
if lastdataset is not None: dataset = dataset.filter( launch_id=lastdataset.launch_id)
dataset = dataset.filter( launch_id=lastdataset.launch_id)
# now = timezone.now() # now = timezone.now()
# epoch = now - datetime.timedelta(minutes=2) # epoch = now - datetime.timedelta(minutes=2)
# dataset = dataset.filter( created_at__range=(epoch, now )) # dataset = dataset.filter( created_at__range=(epoch, now ))