update mailbox trafic

orchestra/contrib/mailboxes/backends.py

@@ -494,7 +494,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
     model = 'mailboxes.Mailbox'
     resource = ServiceMonitor.TRAFFIC
     verbose_name = _("Postfix-Mailscanner traffic")
-    script_executable = '/usr/bin/python'
+    script_executable = '/usr/bin/python3'
     monthly_sum_old_values = True
     doc_settings = (settings,
         ('MAILBOXES_MAIL_LOG_PATH',)
@@ -524,6 +524,10 @@ class PostfixMailscannerTraffic(ServiceMonitor):
             end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
             months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
             months = dict((m, '%02d' % n) for n, m in enumerate(months, 1))
+            users = {{}}
+            sends = {{}}                        
+            register_imap_traffic = False
+            register_pop_traffic = False
             
             def inside_period(month, day, time, ini_date):
                 global months
@@ -539,91 +543,109 @@ class PostfixMailscannerTraffic(ServiceMonitor):
                 date += time.replace(':', '')
                 return ini_date < int(date) < end_date
                        
-            users = {{}}
+            def search_username(pattern, users, line):
-            delivers = {{}}
+                match = pattern.search(line)
-            reverse = {{}}
+                if not match:
+                    return None
+                username = match.groups(1)[0]
+                if username not in users.keys():
+                    return None
+                return username
+
+            def search_size(line, users, username, pattern):
+                month, day, time, req_id  = line.split()[:4]
+                if inside_period(month, day, time, users[username][0]):
+                    group = req_id.split('<')[-1][:-2]
+                    matches = pattern.search(line)
+                    if not matches:
+                        return None, None
+                    return group, matches
+                return None, None
             
             def prepare(object_id, mailbox, ini_date):
                 global users
-                global delivers
+                global sends
-                global reverse
                 ini_date = to_local_timezone(ini_date)
                 ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
                 users[mailbox] = (ini_date, object_id)
-                delivers[mailbox] = set()
+                sends[mailbox] = {{}}
-                reverse[mailbox] = set()
+            
+            def monitor(users, sends, maillogs):
+                grupos = []
+                sasl_username_pattern = re.compile(r'sasl_username=([a-zA-Z0-9\.\-_]+)')
+                size_pattern = re.compile(r'size=(\d+),')
+                
+                pop_username_pattern = re.compile(r' pop3\(([^)].*)\)')
+                pop_size_pattern = re.compile(r'size=(\d+)')
+
+                imap_username_pattern = re.compile(r' imap\(([^)].*)\)')
+                imap_size_pattern = re.compile(r"in=(\d+) out=(\d+)")
                 
-            def monitor(users, delivers, reverse, maillogs):
-                targets = {{}}
-                counter = {{}}
-                user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
                 for maillog in maillogs:
                     try:
                         with open(maillog, 'r') as maillog:
                             for line in maillog.readlines():
                                 # Only search for Authenticated sendings
-                                if '(Authenticated sender: ' in line:
+                                if 'sasl_username=' in line:
-                                    username = user_regex.search(line).groups()[0]
+                                    # si el usuario es uno de los elegidos y el rango de tiempo es correcto
-                                    try:
+                                    # recoge el id de grupo
-                                        sender = users[username]
+                                    username = search_username(sasl_username_pattern, users, line)
-                                    except KeyError:
+                                    if username is None:
                                         continue
+                                    
+                                    month, day, time, __, __, req_id  = line.split()[:6]
+                                    if inside_period(month, day, time, users[username][0]):
+                                        group = req_id[:-1]
+                                        sends[username][group] = 0
+                                        grupos.append(group)
                                 else:
-                                        month, day, time, __, proc, id = line.split()[:6]
+                                    # busca el size de envios donde se alla anadido el groupID anteriormente, 
-                                        if inside_period(month, day, time, sender[0]):
+                                    # una vez encontrado borra el groupID
-                                            # Add new email
+                                    for id in grupos:
-                                            delivers[id[:-1]] = username
+                                        if id in line:
-                                # Look for a MailScanner requeue ID
+                                            match = size_pattern.search(line)
-                                elif ' Requeue: ' in line:
+                                            if not match:
-                                    id, __, req_id = line.split()[6:9]
-                                    id = id.split('.')[0]
-                                    try:
-                                        username = delivers[id]
-                                    except KeyError:
-                                        pass
-                                    else:
-                                        targets[req_id] = (username, 0)
-                                        reverse[username].add(req_id)
-                                # Look for the mail size and count the number of recipients of each email
-                                else:
-                                    try:
-                                        month, day, time, __, proc, req_id, __, msize = line.split()[:8]
-                                    except ValueError:
-                                        # not interested in this line
                                                 continue
-                                    if proc.startswith('postfix/'):
+                                            for k, v in sends.items():
-                                        req_id = req_id[:-1]
+                                                if id in sends[k].keys():
-                                        if msize.startswith('size='):
+                                                    sends[k][id] += int(match.groups(1)[0])
-                                            try:
+                                            grupos.remove(id)
-                                                target = targets[req_id]
+                                    
-                                            except KeyError:
+                                # pop trafic
-                                                pass
+                                if register_pop_traffic:
-                                            else:
+                                    if 'pop3(' in line and 'size' in line:
-                                                targets[req_id] = (target[0], int(msize[5:-1]))
+                                        username = search_username(pop_username_pattern, users, line)
-                                        elif proc.startswith('postfix/smtp'):
+                                        if username is None:
-                                            try:
+                                            continue
-                                                target = targets[req_id]
+                                        group, matches = search_size(line, users, username, pop_size_pattern)
-                                            except KeyError:
+                                        if group is not None and matches is not None :
-                                                pass
+                                            sends[username][group] = int(matches.groups(1)[0])
-                                            else:
+                                
-                                                if inside_period(month, day, time, users[target[0]][0]):
+                                # imap trafic
-                                                    try:
+                                if register_imap_traffic:
-                                                        counter[req_id] += 1
+                                    if 'imap(' in line and 'out=' in line:
-                                                    except KeyError:
+                                        username = search_username(imap_username_pattern, users, line)
-                                                        counter[req_id] = 1
+                                        if username is None:
+                                            continue
+                                        group, matches = search_size(line, users, username, imap_size_pattern)
+                                        if group is not None and matches is not None :
+                                            value = int(matches.group(1)) + int(matches.group(2)) 
+                                            sends[username][group] = value
+
                     except IOError as e:
                         sys.stderr.write(str(e)+'\\n')
                     
-                for username, opts in users.iteritems():
+                # devolver la sumatoria de valores a orchestra (id_user,  size)
-                    size = 0
+                for username, opts in users.items():
-                    for req_id in reverse[username]:
+                    total_size = 0
-                        size += targets[req_id][1] * counter.get(req_id, 0)
+                    for size in sends[username].values():
-                    print opts[1], size
+                        total_size += size
+                    print(f"{{opts[1]}} {{total_size}}")
             """).format(**context)
         )
     
     def commit(self):
-        self.append('monitor(users, delivers, reverse, maillogs)')
+        self.append('monitor(users, sends, maillogs)')
     
     def monitor(self, mailbox):
         context = self.get_context(mailbox)

orchestra/contrib/resources/aggregations.py

@@ -34,6 +34,7 @@ class Last(Aggregation):
     def filter(self, dataset, date=None):
 
         lastdataset = dataset.order_by('-id').first()
+        if lastdataset is not  None:
             dataset = dataset.filter( launch_id=lastdataset.launch_id)
         # now = timezone.now()
         # epoch = now - datetime.timedelta(minutes=2)