|
Pierre-Yves Chibon |
623e66 |
#!/usr/bin/env python
|
|
Pierre-Yves Chibon |
623e66 |
# -*- coding: utf-8 -*-
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
"""
|
|
Pierre-Yves Chibon |
623e66 |
(c) 2016 - Copyright Red Hat Inc
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
Authors:
|
|
Pierre-Yves Chibon |
623e66 |
Pierre-Yves Chibon <pingou@pingoured.fr></pingou@pingoured.fr>
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
This server listens to message sent via redis post commits and log the
|
|
Pierre-Yves Chibon |
623e66 |
user's activity in the database.
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
Using this mechanism, we no longer need to block the git push until all the
|
|
Pierre-Yves Chibon |
623e66 |
activity has been logged (which is you push the kernel tree for the first
|
|
Pierre-Yves Chibon |
623e66 |
time can be really time-consuming).
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
"""
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
import json
|
|
Pierre-Yves Chibon |
623e66 |
import logging
|
|
Pierre-Yves Chibon |
623e66 |
import os
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
import trollius
|
|
Pierre-Yves Chibon |
623e66 |
import trollius_redis
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
e2027b |
_log = logging.getLogger(__name__)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
if 'PAGURE_CONFIG' not in os.environ \
|
|
Pierre-Yves Chibon |
623e66 |
and os.path.exists('/etc/pagure/pagure.cfg'):
|
|
Pierre-Yves Chibon |
623e66 |
print 'Using configuration file `/etc/pagure/pagure.cfg`'
|
|
Pierre-Yves Chibon |
623e66 |
os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
import pagure
|
|
Pierre-Yves Chibon |
623e66 |
import pagure.lib
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
@trollius.coroutine
|
|
Pierre-Yves Chibon |
623e66 |
def handle_messages():
|
|
Pierre-Yves Chibon |
623e66 |
''' Handles connecting to redis and acting upon messages received.
|
|
Pierre-Yves Chibon |
76d5e0 |
In this case, it means logging into the DB the commits specified in the
|
|
Pierre-Yves Chibon |
76d5e0 |
message for the specified repo.
|
|
Pierre-Yves Chibon |
76d5e0 |
|
|
Pierre-Yves Chibon |
76d5e0 |
The currently accepted message format looks like:
|
|
Pierre-Yves Chibon |
76d5e0 |
|
|
Pierre-Yves Chibon |
76d5e0 |
::
|
|
Pierre-Yves Chibon |
76d5e0 |
|
|
Pierre-Yves Chibon |
76d5e0 |
{
|
|
Pierre-Yves Chibon |
76d5e0 |
"project": {
|
|
Pierre-Yves Chibon |
76d5e0 |
"name": "foo",
|
|
Pierre-Yves Chibon |
76d5e0 |
"namespace": null,
|
|
Pierre-Yves Chibon |
76d5e0 |
"parent": null,
|
|
Pierre-Yves Chibon |
76d5e0 |
"username": {
|
|
Pierre-Yves Chibon |
76d5e0 |
"name": "user"
|
|
Pierre-Yves Chibon |
76d5e0 |
}
|
|
Pierre-Yves Chibon |
76d5e0 |
},
|
|
Pierre-Yves Chibon |
76d5e0 |
"abspath": "/srv/git/repositories/pagure.git",
|
|
Pierre-Yves Chibon |
76d5e0 |
"commits": [
|
|
Pierre-Yves Chibon |
76d5e0 |
"b7b4059c44d692d7df3227ce58ce01191e5407bd",
|
|
Pierre-Yves Chibon |
76d5e0 |
"f8d0899bb6654590ffdef66b539fd3b8cf873b35",
|
|
Pierre-Yves Chibon |
76d5e0 |
"9b6fdc48d3edab82d3de28953271ea52b0a96117"
|
|
Pierre-Yves Chibon |
76d5e0 |
]
|
|
Pierre-Yves Chibon |
76d5e0 |
}
|
|
Pierre-Yves Chibon |
76d5e0 |
|
|
Pierre-Yves Chibon |
623e66 |
'''
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0')
|
|
Pierre-Yves Chibon |
623e66 |
port = pagure.APP.config.get('REDIS_PORT', 6379)
|
|
Pierre-Yves Chibon |
623e66 |
dbname = pagure.APP.config.get('REDIS_DB', 0)
|
|
Pierre-Yves Chibon |
623e66 |
connection = yield trollius.From(trollius_redis.Connection.create(
|
|
Pierre-Yves Chibon |
623e66 |
host=host, port=port, db=dbname))
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
# Create subscriber.
|
|
Pierre-Yves Chibon |
623e66 |
subscriber = yield trollius.From(connection.start_subscribe())
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
# Subscribe to channel.
|
|
Pierre-Yves Chibon |
623e66 |
yield trollius.From(subscriber.subscribe(['pagure.logcom']))
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
# Inside a while loop, wait for incoming events.
|
|
Pierre-Yves Chibon |
623e66 |
while True:
|
|
Pierre-Yves Chibon |
623e66 |
reply = yield trollius.From(subscriber.next_published())
|
|
Pierre-Yves Chibon |
e2027b |
_log.info(
|
|
Pierre-Yves Chibon |
623e66 |
'Received: %s on channel: %s',
|
|
Pierre-Yves Chibon |
623e66 |
repr(reply.value), reply.channel)
|
|
Pierre-Yves Chibon |
623e66 |
data = json.loads(reply.value)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
commits = data['commits']
|
|
Pierre-Yves Chibon |
623e66 |
abspath = data['abspath']
|
|
Pierre-Yves Chibon |
623e66 |
repo = data['project']['name']
|
|
Pierre-Yves Chibon |
623e66 |
username = data['project']['username']['name'] \
|
|
Pierre-Yves Chibon |
623e66 |
if data['project']['parent'] else None
|
|
Pierre-Yves Chibon |
623e66 |
namespace = data['project']['namespace']
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
session = pagure.lib.create_session(pagure.APP.config['DB_URL'])
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
e2027b |
_log.info('Looking for project: %s%s of %s',
|
|
Pierre-Yves Chibon |
623e66 |
'%s/' % namespacerepo if namespace else '',
|
|
Pierre-Yves Chibon |
623e66 |
repo, username)
|
|
Farhaan Bukhsh |
e498a1 |
project = pagure.lib._get_project(
|
|
Pierre-Yves Chibon |
623e66 |
pagure.SESSION, repo, user=username, namespace=namespace)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
if not project:
|
|
Pierre-Yves Chibon |
e2027b |
_log.info('No project found')
|
|
Pierre-Yves Chibon |
623e66 |
continue
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
e2027b |
_log.info('Found project: %s', project.fullname)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
e2027b |
_log.info('Processing %s commits in %s', len(commits), abspath)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
pagure.lib.git.log_commits_to_db(
|
|
Pierre-Yves Chibon |
79b62c |
session, project, commits, abspath)
|
|
Pierre-Yves Chibon |
79b62c |
|
|
Pierre-Yves Chibon |
79b62c |
try:
|
|
Pierre-Yves Chibon |
79b62c |
session.commit()
|
|
Pierre-Yves Chibon |
79b62c |
except SQLAlchemyError as err: # pragma: no cover
|
|
Pierre-Yves Chibon |
79b62c |
session.rollback()
|
|
Pierre-Yves Chibon |
79b62c |
finally:
|
|
Pierre-Yves Chibon |
79b62c |
session.close()
|
|
Pierre-Yves Chibon |
e2027b |
_log.info('Ready for another')
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
def main():
|
|
Pierre-Yves Chibon |
623e66 |
''' Start the main async loop. '''
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
try:
|
|
Pierre-Yves Chibon |
623e66 |
loop = trollius.get_event_loop()
|
|
Pierre-Yves Chibon |
623e66 |
tasks = [
|
|
Pierre-Yves Chibon |
623e66 |
trollius.async(handle_messages()),
|
|
Pierre-Yves Chibon |
623e66 |
]
|
|
Pierre-Yves Chibon |
623e66 |
loop.run_until_complete(trollius.wait(tasks))
|
|
Pierre-Yves Chibon |
623e66 |
loop.run_forever()
|
|
Pierre-Yves Chibon |
623e66 |
except KeyboardInterrupt:
|
|
Pierre-Yves Chibon |
623e66 |
pass
|
|
Pierre-Yves Chibon |
623e66 |
except trollius.ConnectionResetError:
|
|
Pierre-Yves Chibon |
623e66 |
pass
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
e2027b |
_log.info("End Connection")
|
|
Pierre-Yves Chibon |
623e66 |
loop.close()
|
|
Pierre-Yves Chibon |
e2027b |
_log.info("End")
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
if __name__ == '__main__':
|
|
Pierre-Yves Chibon |
623e66 |
formatter = logging.Formatter(
|
|
Pierre-Yves Chibon |
623e66 |
"%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s")
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
logging.basicConfig(level=logging.DEBUG)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
# setup console logging
|
|
Pierre-Yves Chibon |
e2027b |
_log.setLevel(logging.DEBUG)
|
|
Pierre-Yves Chibon |
623e66 |
shellhandler = logging.StreamHandler()
|
|
Pierre-Yves Chibon |
623e66 |
shellhandler.setLevel(logging.DEBUG)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
aslog = logging.getLogger("asyncio")
|
|
Pierre-Yves Chibon |
623e66 |
aslog.setLevel(logging.DEBUG)
|
|
Pierre-Yves Chibon |
623e66 |
aslog = logging.getLogger("trollius")
|
|
Pierre-Yves Chibon |
623e66 |
aslog.setLevel(logging.DEBUG)
|
|
Pierre-Yves Chibon |
623e66 |
|
|
Pierre-Yves Chibon |
623e66 |
shellhandler.setFormatter(formatter)
|
|
Pierre-Yves Chibon |
e2027b |
_log.addHandler(shellhandler)
|
|
Pierre-Yves Chibon |
623e66 |
main()
|