diff --git a/pagure-ci/README.rst b/pagure-ci/README.rst new file mode 100644 index 0000000..9c88d0a --- /dev/null +++ b/pagure-ci/README.rst @@ -0,0 +1,118 @@ +Pagure CI +========= + +This is to setup Pagure CI for development. It is assumed that all the +dependencies are resolved. It is advised to use a virtual envivironment +for development. + + * Run:: + + python setup.py develop + + +Now in pagureCI/consumer.py add the following elements in `topic` list + +:: + + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, +after you login to your Jenkins Instance. + + +* Go to Manage Jenkins -> Configuire Global Security and under that select + 'Project-based Matrix Authorization Strategy' + +* Add a user and give all the permission to that user. + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name you + filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: Job Finalized + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token + that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +* After all the configuration done, go to the dev instance of pagure running + and under project settings in `Plugin` select Pagure CI and fill the appropriate + information. Which on submiting should give you a POST url. + +* Copy and paste the URL in the Notification section under the Jenkins project + you want the CI to work, + + +Get It Running: +=============== + +In one terminal window run: + +:: + + fedmsg-relay + +Another window: + +:: + + fedmsg-hub + +* Now clone the project locally and make a branch. make some changes and push it + to the repo and try to make a PR. You will notice if everything works fine a lot + of logs in the server console and `build fail` flag on the PR. + Build fail because there is no git server running. diff --git a/pagure-ci/__init__.py b/pagure-ci/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure-ci/__init__.py diff --git a/pagure-ci/consumer.py b/pagure-ci/consumer.py new file mode 100644 index 0000000..ea97f30 --- /dev/null +++ b/pagure-ci/consumer.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +import pagure.lib +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +from pagure import APP, SESSION +import pagure.exceptions +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + ''' Integrates Jenkins with Pagure. ''' + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + + def consume(self, msg): + ''' Pagure CI consumer which consumes message from + fedmsg and triggers Jenkins build. + ''' + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + except pagure.exceptions.HookInactiveException as exc: + self.log.info('Hook Inactive for project %r', str(exc)) + + def trigger_build(self, msg): + ''' Triggers or requests to start a build in Jenkins. ''' + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + ''' Extracts the information from the build and flag the pull-request. ''' + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + ''' Formats the URL for pagure repo. ''' + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=APP.config['APP_URL'], + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + ''' Returns Rebase if the Pull-request is rebased. ''' + if msg['pullrequest']['status'] != 'Open': + return False + try: + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/pagure-ci/pagure_ci_server.py b/pagure-ci/pagure_ci_server.py new file mode 100644 index 0000000..8b43c65 --- /dev/null +++ b/pagure-ci/pagure_ci_server.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and send the corresponding +web-hook request. + +Using this mechanism, we no longer block the main application if the +receiving end is offline or so. + +""" + +import datetime +import hashlib +import hmac +import json +import logging +import os +import requests +import time +import uuid + +import six +import trollius +import trollius_redis + +from kitchen.text.converters import to_bytes + + +log = logging.getLogger(__name__) + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib +from pagure.exceptions import PagureEvException + +_i = 0 + + +@trollius.coroutine +def handle_messages(): + connection = yield trollius.From(trollius_redis.Connection.create( + host='0.0.0.0', port=6379, db=0)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.ci'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + pr_id = data['pr']['id'] + project = data['pr']['project']['name'] + branch = data['pr']['branch_from'] + + username = None + projectname = data['pr']['project'] + if data['pr'].get('parent'): + username, data['pr']['project']['user']['user'] + + project = pagure.lib.get_project( + session=pagure.SESSION, name=projectname, user=username) + + if not project: + log.warning( + 'No project could be found from the message %s' % data) + continue + + repo = data['pr'].get('remote_git') + if not repo: + base = pagure.APP.config['APP_URL'] + if base.endswith('/'): + base[:-1] + base += '/%s' % project.path + + log.info("Trigger on %s PR #%s from %s: %s", + project.fullname, pr_id, repo, branch) + + url = project.ci_hook[0].ci_url + if url.endswith('/'): + url = url[:-1] + + if data['ci_type'] == 'jenkins': + url += '/buildWithParameters' + request.post( + url, + data={ + 'token': cfg.jenkins_token, + 'cause': pr_id, + 'REPO': repo.fullname, + 'BRANCH': branch + } + ) + else: + log.warning('Un-supported CI type') + + +def main(): + server = None + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + log.info("End Connection") + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/pagureCI/README.rst b/pagureCI/README.rst deleted file mode 100644 index 9c88d0a..0000000 --- a/pagureCI/README.rst +++ /dev/null @@ -1,118 +0,0 @@ -Pagure CI -========= - -This is to setup Pagure CI for development. It is assumed that all the -dependencies are resolved. It is advised to use a virtual envivironment -for development. - - * Run:: - - python setup.py develop - - -Now in pagureCI/consumer.py add the following elements in `topic` list - -:: - - 'org.fedoraproject.dev.pagure.pull-request.new', - 'org.fedoraproject.dev.pagure.pull-request.comment.added', - - -Configuring Jenkins -=================== - -Jenkins configuration is the most important part of how the Pagure CI works, -after you login to your Jenkins Instance. - - -* Go to Manage Jenkins -> Configuire Global Security and under that select - 'Project-based Matrix Authorization Strategy' - -* Add a user and give all the permission to that user. - -* Download the following plugins: - -:: - - Build Authorization Root Plugin - Git Plugins - Notification Plugin - - -* Click on the New Item - -* Select Freestyle Project - -* Click OK and enter the name of the project, make sure the project name you - filled in the Pagure CI form should match the name you entered here. - -* Under 'Job Notification' click 'Add Endpoint' - -* Fields in Endpoint will be : - -:: - - FORMAT: JSON - PROTOCOL: HTTP - EVENT: Job Finalized - URL: - TIMEOUT: 3000 - LOG: 1 - -* Tick the build is parameterized - -* From the Add Parameter drop down select String Parameter - -* Two string parameters need to be created REPO and BRANCH - -* Source Code Management select Git and give the URL of the pagure project - -* Under Build Trigger click on Trigger build remotely and give the same token - that you gave in the Pagure CI form. - -* Under Build -> Add build step -> Execute Shell - -* In the box given enter the shell steps you want for testing your project. - - -Example Script - -:: - - if [ -n "$REPO" -a -n "$BRANCH" ]; then - git remote rm proposed || true - git remote add proposed "$REPO" - git fetch proposed - git checkout origin/master - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - git merge --no-ff "proposed/$BRANCH" -m "Merge PR" - fi - -* After all the configuration done, go to the dev instance of pagure running - and under project settings in `Plugin` select Pagure CI and fill the appropriate - information. Which on submiting should give you a POST url. - -* Copy and paste the URL in the Notification section under the Jenkins project - you want the CI to work, - - -Get It Running: -=============== - -In one terminal window run: - -:: - - fedmsg-relay - -Another window: - -:: - - fedmsg-hub - -* Now clone the project locally and make a branch. make some changes and push it - to the repo and try to make a PR. You will notice if everything works fine a lot - of logs in the server console and `build fail` flag on the PR. - Build fail because there is no git server running. diff --git a/pagureCI/__init__.py b/pagureCI/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagureCI/__init__.py +++ /dev/null diff --git a/pagureCI/consumer.py b/pagureCI/consumer.py deleted file mode 100644 index ea97f30..0000000 --- a/pagureCI/consumer.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -import fedmsg.consumers -from pagure.hooks import jenkins_hook -import pagure.lib -from pagure.lib import pagure_ci -from pagure.lib.model import BASE, Project, User -from pagure import APP, SESSION -import pagure.exceptions -PAGURE_MAIN_REPO = '{base}{name}.git' -PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' - - -class Integrator(fedmsg.consumers.FedmsgConsumer): - ''' Integrates Jenkins with Pagure. ''' - topic = [ - 'io.pagure.prod.pagure.pull-request.comment.added', - 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.prod.jenkins.build', - ] - - config_key = 'integrator.enabled' - - - def __init__(self, hub): - super(Integrator, self).__init__(hub) - - def consume(self, msg): - ''' Pagure CI consumer which consumes message from - fedmsg and triggers Jenkins build. - ''' - topic, msg = msg['topic'], msg['body'] - self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) - msg = msg['msg'] - try: - if topic.endswith('.pull-request.comment.added'): - if is_rebase(msg): - self.trigger_build(msg) - elif topic.endswith('.pull-request.new'): - self.trigger_build(msg) - else: - self.process_build(msg) - except jenkins_hook.ConfigNotFound as exc: - self.log.info('Unconfigured project %r', str(exc)) - except pagure.exceptions.HookInactiveException as exc: - self.log.info('Hook Inactive for project %r', str(exc)) - - def trigger_build(self, msg): - ''' Triggers or requests to start a build in Jenkins. ''' - pr_id = msg['pullrequest']['id'] - project = msg['pullrequest']['project']['name'] - branch = msg['pullrequest']['branch_from'] - - for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): - repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - self.log.info("Trigger on %s PR #%s from %s: %s", - project, pr_id, repo, branch) - - pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) - - def process_build(self, msg): - ''' Extracts the information from the build and flag the pull-request. ''' - for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): - pagure_ci.process_build(self.log, cfg, msg['build']) - - -def get_repo(cfg, msg): - ''' Formats the URL for pagure repo. ''' - url = PAGURE_MAIN_REPO - if msg['pullrequest']['repo_from']['parent']: - url = PAGURE_FORK_REPO - return url.format( - base=APP.config['APP_URL'], - user=msg['pullrequest']['repo_from']['user']['name'], - name=msg['pullrequest']['repo_from']['name']) - - -def is_rebase(msg): - ''' Returns Rebase if the Pull-request is rebased. ''' - if msg['pullrequest']['status'] != 'Open': - return False - try: - return msg['pullrequest']['comments'][-1]['notification'] - except (IndexError, KeyError): - return False diff --git a/pagureCI/pagure_ci_server.py b/pagureCI/pagure_ci_server.py deleted file mode 100644 index 8b43c65..0000000 --- a/pagureCI/pagure_ci_server.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - (c) 2016 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -This server listens to message sent via redis and send the corresponding -web-hook request. - -Using this mechanism, we no longer block the main application if the -receiving end is offline or so. - -""" - -import datetime -import hashlib -import hmac -import json -import logging -import os -import requests -import time -import uuid - -import six -import trollius -import trollius_redis - -from kitchen.text.converters import to_bytes - - -log = logging.getLogger(__name__) - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib -from pagure.exceptions import PagureEvException - -_i = 0 - - -@trollius.coroutine -def handle_messages(): - connection = yield trollius.From(trollius_redis.Connection.create( - host='0.0.0.0', port=6379, db=0)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.ci'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - - pr_id = data['pr']['id'] - project = data['pr']['project']['name'] - branch = data['pr']['branch_from'] - - username = None - projectname = data['pr']['project'] - if data['pr'].get('parent'): - username, data['pr']['project']['user']['user'] - - project = pagure.lib.get_project( - session=pagure.SESSION, name=projectname, user=username) - - if not project: - log.warning( - 'No project could be found from the message %s' % data) - continue - - repo = data['pr'].get('remote_git') - if not repo: - base = pagure.APP.config['APP_URL'] - if base.endswith('/'): - base[:-1] - base += '/%s' % project.path - - log.info("Trigger on %s PR #%s from %s: %s", - project.fullname, pr_id, repo, branch) - - url = project.ci_hook[0].ci_url - if url.endswith('/'): - url = url[:-1] - - if data['ci_type'] == 'jenkins': - url += '/buildWithParameters' - request.post( - url, - data={ - 'token': cfg.jenkins_token, - 'cause': pr_id, - 'REPO': repo.fullname, - 'BRANCH': branch - } - ) - else: - log.warning('Un-supported CI type') - - -def main(): - server = None - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - log.info("End Connection") - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main()