From 23eab50d5b00e964c0a9e8abc4f382d1324c8b9f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 13 2017 21:01:30 +0000 Subject: Rename webhook-server into pagure-webhook This will make this service easier to identify and more consistent with the other services we ship. --- diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst index 8e5e0b0..42531e9 100644 --- a/doc/install_webhooks.rst +++ b/doc/install_webhooks.rst @@ -29,9 +29,9 @@ Configure your system +----------------------------------------------+----------------------------------------------------------+ | Source | Destination | +==============================================+==========================================================+ -| ``webhook-server/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | +| ``pagure-webhook/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | +----------------------------------------------+----------------------------------------------------------+ -| ``webhook-server/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | +| ``pagure-webhook/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | +----------------------------------------------+----------------------------------------------------------+ The first file is the script of the web-hook server itself. diff --git a/files/pagure.spec b/files/pagure.spec index 0cf7e9c..766c80f 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -254,9 +254,9 @@ install -m 644 ev-server/pagure_ev.service \ # Install the web-hook mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook -install -m 755 webhook-server/pagure-webhook-server.py \ +install -m 755 pagure-webhook/pagure-webhook-server.py \ $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook/pagure-webhook-server.py -install -m 644 webhook-server/pagure_webhook.service \ +install -m 644 pagure-webhook/pagure_webhook.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_webhook.service # Install the ci service diff --git a/pagure-webhook/pagure-webhook-server.py b/pagure-webhook/pagure-webhook-server.py new file mode 100644 index 0000000..cbcb1dc --- /dev/null +++ b/pagure-webhook/pagure-webhook-server.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +""" + (c) 2015 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and send the corresponding +web-hook request. + +Using this mechanism, we no longer block the main application if the +receiving end is offline or so. + +""" + +import datetime +import hashlib +import hmac +import json +import logging +import os +import requests +import time +import uuid + +import six +import trollius +import trollius_redis + +from kitchen.text.converters import to_bytes + + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib +from pagure.exceptions import PagureEvException + +_i = 0 + + +def call_web_hooks(project, topic, msg, urls): + ''' Sends the web-hook notification. ''' + log.info( + "Processing project: %s - topic: %s", project.fullname, topic) + log.debug('msg: %s', msg) + + # Send web-hooks notification + global _i + _i += 1 + year = datetime.datetime.now().year + if isinstance(topic, six.text_type): + topic = to_bytes(topic, encoding='utf8', nonstring="passthru") + msg['pagure_instance'] = pagure.APP.config['APP_URL'] + msg['project_fullname'] = project.fullname + msg = dict( + topic=topic.decode('utf-8'), + msg=msg, + timestamp=int(time.time()), + msg_id=str(year) + '-' + str(uuid.uuid4()), + i=_i, + ) + + content = json.dumps(msg) + hashhex = hmac.new( + str(project.hook_token), content, hashlib.sha1).hexdigest() + hashhex256 = hmac.new( + str(project.hook_token), content, hashlib.sha256).hexdigest() + headers = { + 'X-Pagure': pagure.APP.config['APP_URL'], + 'X-Pagure-project': project.fullname, + 'X-Pagure-Signature': hashhex, + 'X-Pagure-Signature-256': hashhex256, + 'X-Pagure-Topic': topic, + 'Content-Type': 'application/json', + } + for url in urls: + url = url.strip() + log.info('Calling url %s' % url) + try: + req = requests.post( + url, + headers=headers, + data=content, + timeout=60, + ) + if not req: + log.info( + 'An error occured while querying: %s - ' + 'Error code: %s' % (url, req.status_code)) + except (requests.exceptions.RequestException, Exception) as err: + log.info( + 'An error occured while querying: %s - Error: %s' % ( + url, err)) + + +@trollius.coroutine +def handle_messages(): + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.hook'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + username = None + if data['project'].startswith('forks'): + username, projectname = data['project'].split('/', 2)[1:] + else: + projectname = data['project'] + + namespace = None + if '/' in projectname: + namespace, projectname = projectname.split('/', 1) + + log.info( + 'Searching %s/%s/%s' % (username, namespace, projectname)) + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + project = pagure.lib._get_project( + session=session, name=projectname, user=username, + namespace=namespace) + if not project: + log.info('No project found with these criteria') + session.close() + continue + urls = project.settings.get('Web-hooks') + session.close() + if not urls: + log.info('No URLs set: %s' % urls) + continue + urls = urls.split('\n') + log.info('Got the project, going to the webhooks') + call_web_hooks(project, data['topic'], data['msg'], urls) + + +def main(): + server = None + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + log.info("End Connection") + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/pagure-webhook/pagure_webhook.service b/pagure-webhook/pagure_webhook.service new file mode 100644 index 0000000..e9d0512 --- /dev/null +++ b/pagure-webhook/pagure_webhook.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure WebHook server (Allowing web-hook notifications) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-webhook/pagure-webhook-server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py deleted file mode 100644 index cbcb1dc..0000000 --- a/webhook-server/pagure-webhook-server.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2015 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -This server listens to message sent via redis and send the corresponding -web-hook request. - -Using this mechanism, we no longer block the main application if the -receiving end is offline or so. - -""" - -import datetime -import hashlib -import hmac -import json -import logging -import os -import requests -import time -import uuid - -import six -import trollius -import trollius_redis - -from kitchen.text.converters import to_bytes - - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib -from pagure.exceptions import PagureEvException - -_i = 0 - - -def call_web_hooks(project, topic, msg, urls): - ''' Sends the web-hook notification. ''' - log.info( - "Processing project: %s - topic: %s", project.fullname, topic) - log.debug('msg: %s', msg) - - # Send web-hooks notification - global _i - _i += 1 - year = datetime.datetime.now().year - if isinstance(topic, six.text_type): - topic = to_bytes(topic, encoding='utf8', nonstring="passthru") - msg['pagure_instance'] = pagure.APP.config['APP_URL'] - msg['project_fullname'] = project.fullname - msg = dict( - topic=topic.decode('utf-8'), - msg=msg, - timestamp=int(time.time()), - msg_id=str(year) + '-' + str(uuid.uuid4()), - i=_i, - ) - - content = json.dumps(msg) - hashhex = hmac.new( - str(project.hook_token), content, hashlib.sha1).hexdigest() - hashhex256 = hmac.new( - str(project.hook_token), content, hashlib.sha256).hexdigest() - headers = { - 'X-Pagure': pagure.APP.config['APP_URL'], - 'X-Pagure-project': project.fullname, - 'X-Pagure-Signature': hashhex, - 'X-Pagure-Signature-256': hashhex256, - 'X-Pagure-Topic': topic, - 'Content-Type': 'application/json', - } - for url in urls: - url = url.strip() - log.info('Calling url %s' % url) - try: - req = requests.post( - url, - headers=headers, - data=content, - timeout=60, - ) - if not req: - log.info( - 'An error occured while querying: %s - ' - 'Error code: %s' % (url, req.status_code)) - except (requests.exceptions.RequestException, Exception) as err: - log.info( - 'An error occured while querying: %s - Error: %s' % ( - url, err)) - - -@trollius.coroutine -def handle_messages(): - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.hook'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - username = None - if data['project'].startswith('forks'): - username, projectname = data['project'].split('/', 2)[1:] - else: - projectname = data['project'] - - namespace = None - if '/' in projectname: - namespace, projectname = projectname.split('/', 1) - - log.info( - 'Searching %s/%s/%s' % (username, namespace, projectname)) - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - project = pagure.lib._get_project( - session=session, name=projectname, user=username, - namespace=namespace) - if not project: - log.info('No project found with these criteria') - session.close() - continue - urls = project.settings.get('Web-hooks') - session.close() - if not urls: - log.info('No URLs set: %s' % urls) - continue - urls = urls.split('\n') - log.info('Got the project, going to the webhooks') - call_web_hooks(project, data['topic'], data['msg'], urls) - - -def main(): - server = None - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - log.info("End Connection") - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main() diff --git a/webhook-server/pagure_webhook.service b/webhook-server/pagure_webhook.service deleted file mode 100644 index e9d0512..0000000 --- a/webhook-server/pagure_webhook.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure WebHook server (Allowing web-hook notifications) -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -ExecStart=/usr/libexec/pagure-webhook/pagure-webhook-server.py -Type=simple -User=git -Group=git -Restart=on-failure - -[Install] -WantedBy=multi-user.target