Blame pagure-milters/comment_email_milter.py

Pierre-Yves Chibon 6e7f9d
#!/usr/bin/env python2
Pierre-Yves Chibon 6e7f9d
# -*- coding: utf-8 -*-
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
# Milter calls methods of your class at milter events.
Pierre-Yves Chibon 6e7f9d
# Return REJECT,TEMPFAIL,ACCEPT to short circuit processing for a message.
Pierre-Yves Chibon 6e7f9d
# You can also add/del recipients, replacebody, add/del headers, etc.
Pierre-Yves Chibon 6e7f9d
Aurélien Bompard dcf6f6
from __future__ import print_function, unicode_literals
Aurélien Bompard dcf6f6
Pierre-Yves Chibon 6e7f9d
import base64
Pierre-Yves Chibon 6e7f9d
import email
Pierre-Yves Chibon 3bcdaf
import hashlib
Pierre-Yves Chibon 6e7f9d
import os
Pierre-Yves Chibon 6e7f9d
import sys
Pierre-Yves Chibon 6e7f9d
import time
Aurélien Bompard 831553
from io import BytesIO
Pierre-Yves Chibon a11348
from multiprocessing import Process as Thread, Queue
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
import Milter
Pierre-Yves Chibon a11348
import requests
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
from Milter.utils import parse_addr
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon b130e5
import pagure
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
if 'PAGURE_CONFIG' not in os.environ \
Pierre-Yves Chibon 6e7f9d
        and os.path.exists('/etc/pagure/pagure.cfg'):
Pierre-Yves Chibon 6e7f9d
    os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon b130e5
logq = Queue(maxsize=4)
Aurélien Bompard 831553
_config = pagure.config.reload_config()
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
def get_email_body(emailobj):
Pierre-Yves Chibon 6e7f9d
    ''' Return the body of the email, preferably in text.
Pierre-Yves Chibon 6e7f9d
    '''
Pierre-Yves Chibon 10cfaa
    def _get_body(emailobj):
Pierre-Yves Chibon 10cfaa
        """ Return the first text/plain body found if the email is multipart
Pierre-Yves Chibon 10cfaa
        or just the regular payload otherwise.
Pierre-Yves Chibon 10cfaa
        """
Pierre-Yves Chibon 10cfaa
        if emailobj.is_multipart():
Pierre-Yves Chibon 10cfaa
            for payload in emailobj.get_payload():
Pierre-Yves Chibon 10cfaa
                # If the message comes with a signature it can be that this
Pierre-Yves Chibon 10cfaa
                # payload itself has multiple parts, so just return the
Pierre-Yves Chibon 10cfaa
                # first one
Pierre-Yves Chibon 10cfaa
                if payload.is_multipart():
Pierre-Yves Chibon 10cfaa
                    return _get_body(payload)
Pierre-Yves Chibon 10cfaa
Pierre-Yves Chibon 10cfaa
                body = payload.get_payload()
Pierre-Yves Chibon 10cfaa
                if payload.get_content_type() == 'text/plain':
Pierre-Yves Chibon 10cfaa
                    return body
Pierre-Yves Chibon 10cfaa
        else:
Pierre-Yves Chibon 10cfaa
            return emailobj.get_payload()
Pierre-Yves Chibon 10cfaa
Pierre-Yves Chibon 10cfaa
    body = _get_body(emailobj)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    enc = emailobj['Content-Transfer-Encoding']
Pierre-Yves Chibon 6e7f9d
    if enc == 'base64':
Pierre-Yves Chibon 6e7f9d
        body = base64.decodestring(body)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    return body
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 98b5c0
def clean_item(item):
Pierre-Yves Chibon 98b5c0
    ''' For an item provided as <item> return the content, if there are no</item>
Pierre-Yves Chibon 98b5c0
    <> then return the string.
Pierre-Yves Chibon 98b5c0
    '''
Pierre-Yves Chibon 98b5c0
    if '<' in item:
Pierre-Yves Chibon 98b5c0
        item = item.split('<')[1]
Pierre-Yves Chibon 98b5c0
    if '>' in item:
Pierre-Yves Chibon 98b5c0
        item = item.split('>')[0]
Pierre-Yves Chibon 98b5c0
Pierre-Yves Chibon 98b5c0
    return item
Pierre-Yves Chibon 98b5c0
Pierre-Yves Chibon 98b5c0
Pierre-Yves Chibon 2ba989
class PagureMilter(Milter.Base):
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    def __init__(self):  # A new instance with each new connection.
Pierre-Yves Chibon 6e7f9d
        self.id = Milter.uniqueID()  # Integer incremented with each call.
Pierre-Yves Chibon 6e7f9d
        self.fp = None
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon d8a338
    def log(self, message):
Pierre-Yves Chibon d8a338
        print(message)
Pierre-Yves Chibon d8a338
        sys.stdout.flush()
Pierre-Yves Chibon d8a338
Pierre-Yves Chibon 6e7f9d
    def envfrom(self, mailfrom, *str):
Pierre-Yves Chibon a6bb9d
        self.log("mail from: %s  -  %s" % (mailfrom, str))
Pierre-Yves Chibon 6e7f9d
        self.fromparms = Milter.dictfromlist(str)
Pierre-Yves Chibon 6e7f9d
        # NOTE: self.fp is only an *internal* copy of message data.  You
Pierre-Yves Chibon 6e7f9d
        # must use addheader, chgheader, replacebody to change the message
Pierre-Yves Chibon 6e7f9d
        # on the MTA.
Aurélien Bompard 831553
        self.fp = BytesIO()
Pierre-Yves Chibon 6e7f9d
        self.canon_from = '@'.join(parse_addr(mailfrom))
Pierre-Yves Chibon 6e7f9d
        self.fp.write('From %s %s\n' % (self.canon_from, time.ctime()))
Pierre-Yves Chibon 6e7f9d
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    @Milter.noreply
Pierre-Yves Chibon 6e7f9d
    def header(self, name, hval):
Pierre-Yves Chibon 837e71
        ''' Headers '''
Pierre-Yves Chibon 837e71
        # add header to buffer
Pierre-Yves Chibon 837e71
        self.fp.write("%s: %s\n" % (name, hval))
Pierre-Yves Chibon 6e7f9d
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    @Milter.noreply
Pierre-Yves Chibon 6e7f9d
    def eoh(self):
Pierre-Yves Chibon 837e71
        ''' End of Headers '''
Pierre-Yves Chibon 837e71
        self.fp.write("\n")
Pierre-Yves Chibon 6e7f9d
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    @Milter.noreply
Pierre-Yves Chibon 6e7f9d
    def body(self, chunk):
Pierre-Yves Chibon 837e71
        ''' Body '''
Pierre-Yves Chibon 6e7f9d
        self.fp.write(chunk)
Pierre-Yves Chibon 6e7f9d
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 5ecf59
    @Milter.noreply
Pierre-Yves Chibon 5ecf59
    def envrcpt(self, to, *str):
Pierre-Yves Chibon 5ecf59
        rcptinfo = to, Milter.dictfromlist(str)
Vadim Rutkovsky 9b6a62
        print(rcptinfo)
Pierre-Yves Chibon 5ecf59
Pierre-Yves Chibon 5ecf59
        return Milter.CONTINUE
Pierre-Yves Chibon 5ecf59
Pierre-Yves Chibon 6e7f9d
    def eom(self):
Pierre-Yves Chibon 837e71
        ''' End of Message '''
Pierre-Yves Chibon 6e7f9d
        self.fp.seek(0)
Pierre-Yves Chibon 6e7f9d
        msg = email.message_from_file(self.fp)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 1ea00a
        msg_id = msg.get('In-Reply-To', None)
Pierre-Yves Chibon 1ea00a
        if msg_id is None:
Pierre-Yves Chibon d8a338
            self.log('No In-Reply-To, keep going')
Pierre-Yves Chibon 1ea00a
            return Milter.CONTINUE
Pierre-Yves Chibon 1ea00a
Pierre-Yves Chibon ff11e4
        # Ensure we don't get extra lines in the message-id
Pierre-Yves Chibon ff11e4
        msg_id = msg_id.split('\n')[0].strip()
Pierre-Yves Chibon ff11e4
Pierre-Yves Chibon d8a338
        self.log('msg-ig %s' % msg_id)
Pierre-Yves Chibon d8a338
        self.log('To %s' % msg['to'])
Pierre-Yves Chibon f6d618
        self.log('Cc %s' % msg.get('cc'))
Pierre-Yves Chibon d8a338
        self.log('From %s' % msg['From'])
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 655827
        # Check the email was sent to the right address
Pierre-Yves Chibon 5a54a4
        email_address = msg['to']
Pierre-Yves Chibon 5a54a4
        if 'reply+' in msg.get('cc', ''):
Pierre-Yves Chibon 5a54a4
            email_address = msg['cc']
Pierre-Yves Chibon a22ec5
        if 'reply+' not in email_address:
Pierre-Yves Chibon 5a54a4
            self.log(
Pierre-Yves Chibon 5a54a4
                'No valid recipient email found in To/Cc: %s'
Pierre-Yves Chibon 5a54a4
                % email_address)
Pierre-Yves Chibon 655827
            return Milter.CONTINUE
Pierre-Yves Chibon 655827
Pierre-Yves Chibon 655827
        # Ensure the user replied to his/her own notification, not that
Pierre-Yves Chibon 655827
        # they are trying to forge their ID into someone else's
Pierre-Yves Chibon b130e5
        salt = _config.get('SALT_EMAIL')
Pierre-Yves Chibon 655827
        from_email = clean_item(msg['From'])
Pierre-Yves Chibon 655827
        try:
Pierre-Yves Chibon 655827
            user = pagure.lib.get_user(pagure.SESSION, from_email)
Pierre-Yves Chibon 655827
        except:
Pierre-Yves Chibon 655827
            self.log(
Pierre-Yves Chibon 655827
                "Could not find an user in the DB associated with %s" %
Pierre-Yves Chibon a22ec5
                from_email)
Pierre-Yves Chibon 655827
            return Milter.CONTINUE
Pierre-Yves Chibon 655827
Pierre-Yves Chibon 655827
        hashes = []
Pierre-Yves Chibon 655827
        for email_obj in user.emails:
Pierre-Yves Chibon 655827
            m = hashlib.sha512('%s%s%s' % (msg_id, salt, email_obj.email))
Pierre-Yves Chibon 655827
            hashes.append(m.hexdigest())
Pierre-Yves Chibon 655827
Pierre-Yves Chibon 5a54a4
        tohash = email_address.split('@')[0].split('+')[-1]
Pierre-Yves Chibon 655827
        if tohash not in hashes:
Pierre-Yves Chibon 655827
            self.log('hash list: %s' % hashes)
Pierre-Yves Chibon 655827
            self.log('tohash:    %s' % tohash)
Pierre-Yves Chibon d8a338
            self.log('Hash does not correspond to the destination')
Pierre-Yves Chibon 3bcdaf
            return Milter.CONTINUE
Pierre-Yves Chibon 3bcdaf
Pierre-Yves Chibon b130e5
        if msg['From'] and msg['From'] == _config.get('FROM_EMAIL'):
Pierre-Yves Chibon d8a338
            self.log("Let's not process the email we send")
Pierre-Yves Chibon f03aa2
            return Milter.CONTINUE
Pierre-Yves Chibon f03aa2
Pierre-Yves Chibon 98b5c0
        msg_id = clean_item(msg_id)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
        if msg_id and '-ticket-' in msg_id:
Pierre-Yves Chibon ff11e4
            self.log('Processing issue')
Pierre-Yves Chibon 6e7f9d
            return self.handle_ticket_email(msg, msg_id)
Pierre-Yves Chibon 6e7f9d
        elif msg_id and '-pull-request-' in msg_id:
Pierre-Yves Chibon ff11e4
            self.log('Processing pull-request')
Pierre-Yves Chibon 6e7f9d
            return self.handle_request_email(msg, msg_id)
Pierre-Yves Chibon 6e7f9d
        else:
Pierre-Yves Chibon d8a338
            self.log('Not a pagure ticket or pull-request email, let it go')
Pierre-Yves Chibon 6e7f9d
            return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    def handle_ticket_email(self, emailobj, msg_id):
Pierre-Yves Chibon 6e7f9d
        ''' Add the email as a comment on a ticket. '''
Pierre-Yves Chibon a22ec5
        uid = msg_id.split('-ticket-')[-1].split('@')[0]
Pierre-Yves Chibon 6e7f9d
        parent_id = None
Pierre-Yves Chibon 6e7f9d
        if '-' in uid:
Pierre-Yves Chibon 6e7f9d
            uid, parent_id = uid.rsplit('-', 1)
Pierre-Yves Chibon 6e7f9d
        if '/' in uid:
Pierre-Yves Chibon 6e7f9d
            uid = uid.split('/')[0]
Pierre-Yves Chibon d8a338
        self.log('uid %s' % uid)
Pierre-Yves Chibon d8a338
        self.log('parent_id %s' % parent_id)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon a11348
        data = {
Pierre-Yves Chibon d02115
            'objid': uid,
Pierre-Yves Chibon a11348
            'comment': get_email_body(emailobj),
Pierre-Yves Chibon a11348
            'useremail': clean_item(emailobj['From']),
Pierre-Yves Chibon a11348
        }
Pierre-Yves Chibon b130e5
        url = _config.get('APP_URL')
Pierre-Yves Chibon 683132
Pierre-Yves Chibon a11348
        if url.endswith('/'):
Pierre-Yves Chibon a11348
            url = url[:-1]
Pierre-Yves Chibon 5555fd
        url = '%s/pv/ticket/comment/' % url
Pierre-Yves Chibon 655827
        self.log('Calling URL: %s' % url)
Pierre-Yves Chibon a11348
        req = requests.put(url, data=data)
Pierre-Yves Chibon d02115
        if req.status_code == 200:
Pierre-Yves Chibon ebeb10
            self.log('Comment added')
Pierre-Yves Chibon d02115
            return Milter.ACCEPT
Pierre-Yves Chibon 5bf36c
        self.log('Could not add the comment to ticket to pagure')
Pierre-Yves Chibon 5bf36c
        self.log(req.text)
Pierre-Yves Chibon 5bf36c
Pierre-Yves Chibon d02115
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
    def handle_request_email(self, emailobj, msg_id):
Pierre-Yves Chibon 6e7f9d
        ''' Add the email as a comment on a request. '''
Pierre-Yves Chibon a22ec5
        uid = msg_id.split('-pull-request-')[-1].split('@')[0]
Pierre-Yves Chibon 6e7f9d
        parent_id = None
Pierre-Yves Chibon 6e7f9d
        if '-' in uid:
Pierre-Yves Chibon 6e7f9d
            uid, parent_id = uid.rsplit('-', 1)
Pierre-Yves Chibon 6e7f9d
        if '/' in uid:
Pierre-Yves Chibon 6e7f9d
            uid = uid.split('/')[0]
Pierre-Yves Chibon d8a338
        self.log('uid %s' % uid)
Pierre-Yves Chibon d8a338
        self.log('parent_id %s' % parent_id)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon a11348
        data = {
Pierre-Yves Chibon d02115
            'objid': uid,
Pierre-Yves Chibon a11348
            'comment': get_email_body(emailobj),
Pierre-Yves Chibon a11348
            'useremail': clean_item(emailobj['From']),
Pierre-Yves Chibon a11348
        }
Pierre-Yves Chibon b130e5
        url = _config.get('APP_URL')
Pierre-Yves Chibon 683132
Pierre-Yves Chibon a11348
        if url.endswith('/'):
Pierre-Yves Chibon a11348
            url = url[:-1]
Pierre-Yves Chibon 5555fd
        url = '%s/pv/pull-request/comment/' % url
Pierre-Yves Chibon 655827
        self.log('Calling URL: %s' % url)
Pierre-Yves Chibon a11348
        req = requests.put(url, data=data)
Pierre-Yves Chibon 5bf36c
        if req.status_code == 200:
Pierre-Yves Chibon 5bf36c
            self.log('Comment added on PR')
Pierre-Yves Chibon 5bf36c
            return Milter.ACCEPT
Pierre-Yves Chibon 5bf36c
        self.log('Could not add the comment to PR to pagure')
Pierre-Yves Chibon 5bf36c
        self.log(req.text)
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 5bf36c
        return Milter.CONTINUE
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
def background():
Pierre-Yves Chibon 6e7f9d
    while True:
Pierre-Yves Chibon 6e7f9d
        t = logq.get()
Pierre-Yves Chibon a22ec5
        if not t:
Pierre-Yves Chibon a22ec5
            break
Pierre-Yves Chibon a22ec5
        msg, id, ts = t
Pierre-Yves Chibon a22ec5
        print("%s [%d]" % (time.strftime(
Pierre-Yves Chibon a22ec5
            '%Y%b%d %H:%M:%S', time.localtime(ts)), id))
Pierre-Yves Chibon 6e7f9d
        # 2005Oct13 02:34:11 [1] msg1 msg2 msg3 ...
Pierre-Yves Chibon a22ec5
        for i in msg:
Pierre-Yves Chibon a22ec5
            print(i,)
Pierre-Yves Chibon 6e7f9d
        print
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 6e7f9d
def main():
Pierre-Yves Chibon 6e7f9d
    bt = Thread(target=background)
Pierre-Yves Chibon 6e7f9d
    bt.start()
Pierre-Yves Chibon 2ba989
    socketname = "/var/run/pagure/paguresock"
Pierre-Yves Chibon 6e7f9d
    timeout = 600
Pierre-Yves Chibon 6e7f9d
    # Register to have the Milter factory create instances of your class:
Pierre-Yves Chibon 2ba989
    Milter.factory = PagureMilter
Vadim Rutkovsky 9b6a62
    print("%s pagure milter startup" % time.strftime('%Y%b%d %H:%M:%S'))
Pierre-Yves Chibon 6e7f9d
    sys.stdout.flush()
Pierre-Yves Chibon 2ba989
    Milter.runmilter("paguremilter", socketname, timeout)
Pierre-Yves Chibon 6e7f9d
    logq.put(None)
Pierre-Yves Chibon 6e7f9d
    bt.join()
Vadim Rutkovsky 9b6a62
    print("%s pagure milter shutdown" % time.strftime('%Y%b%d %H:%M:%S'))
Pierre-Yves Chibon 6e7f9d
Pierre-Yves Chibon 98b5c0
Pierre-Yves Chibon 6e7f9d
if __name__ == "__main__":
Pierre-Yves Chibon 6e7f9d
    main()