Blame pagure-logcom/pagure_logcom_server.py

Pierre-Yves Chibon 623e66
#!/usr/bin/env python
Pierre-Yves Chibon 623e66
# -*- coding: utf-8 -*-
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
"""
Pierre-Yves Chibon 623e66
 (c) 2016 - Copyright Red Hat Inc
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
 Authors:
Pierre-Yves Chibon 623e66
   Pierre-Yves Chibon <pingou@pingoured.fr></pingou@pingoured.fr>
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
This server listens to message sent via redis post commits and log the
Pierre-Yves Chibon 623e66
user's activity in the database.
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Using this mechanism, we no longer need to block the git push until all the
Pierre-Yves Chibon 623e66
activity has been logged (which is you push the kernel tree for the first
Pierre-Yves Chibon 623e66
time can be really time-consuming).
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
"""
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
import json
Pierre-Yves Chibon 623e66
import logging
Pierre-Yves Chibon 623e66
import os
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
import trollius
Pierre-Yves Chibon 623e66
import trollius_redis
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
_log = logging.getLogger(__name__)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
if 'PAGURE_CONFIG' not in os.environ \
Pierre-Yves Chibon 623e66
        and os.path.exists('/etc/pagure/pagure.cfg'):
Pierre-Yves Chibon 623e66
    print 'Using configuration file `/etc/pagure/pagure.cfg`'
Pierre-Yves Chibon 623e66
    os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
import pagure
Pierre-Yves Chibon 623e66
import pagure.lib
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
@trollius.coroutine
Pierre-Yves Chibon 623e66
def handle_messages():
Pierre-Yves Chibon 623e66
    ''' Handles connecting to redis and acting upon messages received.
Pierre-Yves Chibon 76d5e0
    In this case, it means logging into the DB the commits specified in the
Pierre-Yves Chibon 76d5e0
    message for the specified repo.
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
    The currently accepted message format looks like:
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
    ::
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 76d5e0
        {
Pierre-Yves Chibon 76d5e0
          "project": {
Pierre-Yves Chibon 76d5e0
            "name": "foo",
Pierre-Yves Chibon 76d5e0
            "namespace": null,
Pierre-Yves Chibon 76d5e0
            "parent": null,
Pierre-Yves Chibon 76d5e0
            "username": {
Pierre-Yves Chibon 76d5e0
              "name": "user"
Pierre-Yves Chibon 76d5e0
            }
Pierre-Yves Chibon 76d5e0
          },
Pierre-Yves Chibon 76d5e0
          "abspath": "/srv/git/repositories/pagure.git",
Pierre-Yves Chibon 76d5e0
          "commits": [
Pierre-Yves Chibon 76d5e0
            "b7b4059c44d692d7df3227ce58ce01191e5407bd",
Pierre-Yves Chibon 76d5e0
            "f8d0899bb6654590ffdef66b539fd3b8cf873b35",
Pierre-Yves Chibon 76d5e0
            "9b6fdc48d3edab82d3de28953271ea52b0a96117"
Pierre-Yves Chibon 76d5e0
          ]
Pierre-Yves Chibon 76d5e0
        }
Pierre-Yves Chibon 76d5e0
Pierre-Yves Chibon 623e66
    '''
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0')
Pierre-Yves Chibon 623e66
    port = pagure.APP.config.get('REDIS_PORT', 6379)
Pierre-Yves Chibon 623e66
    dbname = pagure.APP.config.get('REDIS_DB', 0)
Pierre-Yves Chibon 623e66
    connection = yield trollius.From(trollius_redis.Connection.create(
Pierre-Yves Chibon 623e66
        host=host, port=port, db=dbname))
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Create subscriber.
Pierre-Yves Chibon 623e66
    subscriber = yield trollius.From(connection.start_subscribe())
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Subscribe to channel.
Pierre-Yves Chibon 623e66
    yield trollius.From(subscriber.subscribe(['pagure.logcom']))
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # Inside a while loop, wait for incoming events.
Pierre-Yves Chibon 623e66
    while True:
Pierre-Yves Chibon 623e66
        reply = yield trollius.From(subscriber.next_published())
Pierre-Yves Chibon e2027b
        _log.info(
Pierre-Yves Chibon 623e66
            'Received: %s on channel: %s',
Pierre-Yves Chibon 623e66
            repr(reply.value), reply.channel)
Pierre-Yves Chibon 623e66
        data = json.loads(reply.value)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        commits = data['commits']
Pierre-Yves Chibon 623e66
        abspath = data['abspath']
Pierre-Yves Chibon 623e66
        repo = data['project']['name']
Pierre-Yves Chibon 623e66
        username = data['project']['username']['name'] \
Pierre-Yves Chibon 623e66
            if data['project']['parent'] else None
Pierre-Yves Chibon 623e66
        namespace = data['project']['namespace']
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        session = pagure.lib.create_session(pagure.APP.config['DB_URL'])
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Looking for project: %s%s of %s',
Pierre-Yves Chibon 623e66
                 '%s/' % namespacerepo if namespace else '',
Pierre-Yves Chibon 623e66
                 repo, username)
Farhaan Bukhsh e498a1
        project = pagure.lib._get_project(
Pierre-Yves Chibon 623e66
            pagure.SESSION, repo, user=username, namespace=namespace)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        if not project:
Pierre-Yves Chibon e2027b
            _log.info('No project found')
Pierre-Yves Chibon 623e66
            continue
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Found project: %s', project.fullname)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
        _log.info('Processing %s commits in %s', len(commits), abspath)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
        pagure.lib.git.log_commits_to_db(
Pierre-Yves Chibon 79b62c
            session, project, commits, abspath)
Pierre-Yves Chibon 79b62c
Pierre-Yves Chibon 79b62c
        try:
Pierre-Yves Chibon 79b62c
            session.commit()
Pierre-Yves Chibon 79b62c
        except SQLAlchemyError as err:  # pragma: no cover
Pierre-Yves Chibon 79b62c
            session.rollback()
Pierre-Yves Chibon 79b62c
        finally:
Pierre-Yves Chibon 79b62c
            session.close()
Pierre-Yves Chibon e2027b
        _log.info('Ready for another')
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
def main():
Pierre-Yves Chibon 623e66
    ''' Start the main async loop. '''
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    try:
Pierre-Yves Chibon 623e66
        loop = trollius.get_event_loop()
Pierre-Yves Chibon 623e66
        tasks = [
Pierre-Yves Chibon 623e66
            trollius.async(handle_messages()),
Pierre-Yves Chibon 623e66
        ]
Pierre-Yves Chibon 623e66
        loop.run_until_complete(trollius.wait(tasks))
Pierre-Yves Chibon 623e66
        loop.run_forever()
Pierre-Yves Chibon 623e66
    except KeyboardInterrupt:
Pierre-Yves Chibon 623e66
        pass
Pierre-Yves Chibon 623e66
    except trollius.ConnectionResetError:
Pierre-Yves Chibon 623e66
        pass
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon e2027b
    _log.info("End Connection")
Pierre-Yves Chibon 623e66
    loop.close()
Pierre-Yves Chibon e2027b
    _log.info("End")
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
if __name__ == '__main__':
Pierre-Yves Chibon 623e66
    formatter = logging.Formatter(
Pierre-Yves Chibon 623e66
        "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s")
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    logging.basicConfig(level=logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    # setup console logging
Pierre-Yves Chibon e2027b
    _log.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
    shellhandler = logging.StreamHandler()
Pierre-Yves Chibon 623e66
    shellhandler.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    aslog = logging.getLogger("asyncio")
Pierre-Yves Chibon 623e66
    aslog.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
    aslog = logging.getLogger("trollius")
Pierre-Yves Chibon 623e66
    aslog.setLevel(logging.DEBUG)
Pierre-Yves Chibon 623e66
Pierre-Yves Chibon 623e66
    shellhandler.setFormatter(formatter)
Pierre-Yves Chibon e2027b
    _log.addHandler(shellhandler)
Pierre-Yves Chibon 623e66
    main()