Blame pagure-logcom/pagure_logcom_server.py

Pierre-Yves Chibon 623e66
#!/usr/bin/env python
Pierre-Yves Chibon 623e66
# -*- coding: utf-8 -*-
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
"""
Pierre-Yves Chibon b130e5
 (c) 2016-2017 - Copyright Red Hat Inc
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
 Authors:
Pierre-Yves Chibon 623e66
   Pierre-Yves Chibon <pingou@pingoured.fr></pingou@pingoured.fr>
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
This server listens to message sent via redis post commits and log the
Pierre-Yves Chibon 623e66
user's activity in the database.
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Using this mechanism, we no longer need to block the git push until all the
Pierre-Yves Chibon 623e66
activity has been logged (which is you push the kernel tree for the first
Pierre-Yves Chibon 623e66
time can be really time-consuming).
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
"""
Pierre-Yves Chibon 623e66
Vadim Rutkovsky 9b6a62
from __future__ import print_function
Pierre-Yves Chibon 623e66
import json
Pierre-Yves Chibon 623e66
import logging
Pierre-Yves Chibon 623e66
import os
Matt Prahl 11e5cf
from sqlalchemy.exc import SQLAlchemyError
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
import trollius
Pierre-Yves Chibon 623e66
import trollius_redis
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon b130e5
import pagure
Pierre-Yves Chibon b130e5
import pagure.lib
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
if 'PAGURE_CONFIG' not in os.environ \
Pierre-Yves Chibon 623e66
        and os.path.exists('/etc/pagure/pagure.cfg'):
Vadim Rutkovsky 9b6a62
    print('Using configuration file `/etc/pagure/pagure.cfg`')
Pierre-Yves Chibon 623e66
    os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon b130e5
_config = pagure.config.config.reload_config()
Pierre-Yves Chibon b130e5
_log = logging.getLogger(__name__)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
@trollius.coroutine
Pierre-Yves Chibon 623e66
def handle_messages():
Pierre-Yves Chibon 623e66
    ''' Handles connecting to redis and acting upon messages received.
Pierre-Yves Chibon 76d5e0
    In this case, it means logging into the DB the commits specified in the
Matt Prahl b49f93
    message for the default repo or sending commit notification emails.
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
    The currently accepted message format looks like:
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
    ::
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
        {
Pierre-Yves Chibon 76d5e0
          "project": {
Pierre-Yves Chibon 76d5e0
            "name": "foo",
Pierre-Yves Chibon 76d5e0
            "namespace": null,
Pierre-Yves Chibon 76d5e0
            "parent": null,
Pierre-Yves Chibon 76d5e0
            "username": {
Pierre-Yves Chibon 76d5e0
              "name": "user"
Pierre-Yves Chibon 76d5e0
            }
Pierre-Yves Chibon 76d5e0
          },
Pierre-Yves Chibon 76d5e0
          "abspath": "/srv/git/repositories/pagure.git",
Pierre-Yves Chibon 76d5e0
          "commits": [
Pierre-Yves Chibon 76d5e0
            "b7b4059c44d692d7df3227ce58ce01191e5407bd",
Pierre-Yves Chibon 76d5e0
            "f8d0899bb6654590ffdef66b539fd3b8cf873b35",
Pierre-Yves Chibon 76d5e0
            "9b6fdc48d3edab82d3de28953271ea52b0a96117"
Matt Prahl b49f93
          ],
Matt Prahl b49f93
          "branch": "master",
Matt Prahl b49f93
          "default_branch": "master"
Pierre-Yves Chibon 76d5e0
        }
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 623e66
    '''
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon b130e5
    host = _config.get('REDIS_HOST', '0.0.0.0')
Pierre-Yves Chibon b130e5
    port = _config.get('REDIS_PORT', 6379)
Pierre-Yves Chibon b130e5
    dbname = _config.get('REDIS_DB', 0)
Pierre-Yves Chibon 623e66
    connection = yield trollius.From(trollius_redis.Connection.create(
Pierre-Yves Chibon 623e66
        host=host, port=port, db=dbname))
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Create subscriber.
Pierre-Yves Chibon 623e66
    subscriber = yield trollius.From(connection.start_subscribe())
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Subscribe to channel.
Pierre-Yves Chibon 623e66
    yield trollius.From(subscriber.subscribe(['pagure.logcom']))
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Inside a while loop, wait for incoming events.
Pierre-Yves Chibon 623e66
    while True:
Pierre-Yves Chibon 623e66
        reply = yield trollius.From(subscriber.next_published())
Pierre-Yves Chibon e2027b
        _log.info(
Pierre-Yves Chibon 623e66
            'Received: %s on channel: %s',
Pierre-Yves Chibon 623e66
            repr(reply.value), reply.channel)
Pierre-Yves Chibon 623e66
        data = json.loads(reply.value)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        commits = data['commits']
Pierre-Yves Chibon 623e66
        abspath = data['abspath']
Patrick Uiterwijk b8366e
        branch = data['branch']
Patrick Uiterwijk b8366e
        default_branch = data['default_branch']
Pierre-Yves Chibon 623e66
        repo = data['project']['name']
Matt Prahl d7d789
        username = data['project']['user']['name'] \
Pierre-Yves Chibon 623e66
            if data['project']['parent'] else None
Pierre-Yves Chibon 623e66
        namespace = data['project']['namespace']
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon b130e5
        session = pagure.lib.create_session(_config['DB_URL'])
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Looking for project: %s%s of %s',
Matt Prahl 9144f3
                 '%s/' % namespace if namespace else '',
Pierre-Yves Chibon 623e66
                 repo, username)
Farhaan Bukhsh e498a1
        project = pagure.lib._get_project(
Pierre-Yves Chibon edbdc9
            pagure.SESSION, repo, user=username, namespace=namespace,
Pierre-Yves Chibon b130e5
            case=_config.get('CASE_SENSITIVE', False))
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        if not project:
Pierre-Yves Chibon e2027b
            _log.info('No project found')
Pierre-Yves Chibon 623e66
            continue
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Found project: %s', project.fullname)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Processing %s commits in %s', len(commits), abspath)
Pierre-Yves Chibon 623e66
Matt Prahl b49f93
        # Only log commits when the branch is the default branch
Matt Prahl b49f93
        if branch == default_branch:
Matt Prahl b49f93
            pagure.lib.git.log_commits_to_db(
Matt Prahl b49f93
                session, project, commits, abspath)
Matt Prahl b49f93
Matt Prahl b49f93
        # Notify subscribed users that there are new commits
Matt Prahl b49f93
        pagure.lib.notify.notify_new_commits(
Matt Prahl b49f93
            abspath, project, branch, commits)
Pierre-Yves Chibon 79b62c
Pierre-Yves Chibon 79b62c
        try:
Pierre-Yves Chibon 79b62c
            session.commit()
Pierre-Yves Chibon 79b62c
        except SQLAlchemyError as err:  # pragma: no cover
Pierre-Yves Chibon 79b62c
            session.rollback()
Pierre-Yves Chibon 79b62c
        finally:
Pierre-Yves Chibon 79b62c
            session.close()
Pierre-Yves Chibon e2027b
        _log.info('Ready for another')
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
def main():
Pierre-Yves Chibon 623e66
    ''' Start the main async loop. '''
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    try:
Pierre-Yves Chibon 623e66
        loop = trollius.get_event_loop()
Pierre-Yves Chibon 623e66
        tasks = [
Pierre-Yves Chibon 623e66
            trollius.async(handle_messages()),
Pierre-Yves Chibon 623e66
        ]
Pierre-Yves Chibon 623e66
        loop.run_until_complete(trollius.wait(tasks))
Pierre-Yves Chibon 623e66
        loop.run_forever()
Pierre-Yves Chibon 623e66
    except KeyboardInterrupt:
Pierre-Yves Chibon 623e66
        pass
Pierre-Yves Chibon 623e66
    except trollius.ConnectionResetError:
Pierre-Yves Chibon 623e66
        pass
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
    _log.info("End Connection")
Pierre-Yves Chibon 623e66
    loop.close()
Pierre-Yves Chibon e2027b
    _log.info("End")
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
if __name__ == '__main__':
Pierre-Yves Chibon 623e66
    formatter = logging.Formatter(
Pierre-Yves Chibon 623e66
        "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s")
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    logging.basicConfig(level=logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # setup console logging
Pierre-Yves Chibon e2027b
    _log.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
    shellhandler = logging.StreamHandler()
Pierre-Yves Chibon 623e66
    shellhandler.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    aslog = logging.getLogger("asyncio")
Pierre-Yves Chibon 623e66
    aslog.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
    aslog = logging.getLogger("trollius")
Pierre-Yves Chibon 623e66
    aslog.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    shellhandler.setFormatter(formatter)
Pierre-Yves Chibon e2027b
    _log.addHandler(shellhandler)
Pierre-Yves Chibon 623e66
    main()