diff --git a/doc/pagure_ci.rst b/doc/pagure_ci.rst new file mode 100644 index 0000000..3714db0 --- /dev/null +++ b/doc/pagure_ci.rst @@ -0,0 +1,115 @@ +========= +Pagure CI +========= + +Pagure CI is a continuous integration tool using which the PR on the projects can be tested and flaged with the status of the build. + +How to enable Pagure CI +======================= + +* Enable the Fedmsg plugin in pagure project setting . This will emit the message to for consumer to consume it. +* Fill in the Pagure CI form with the required details. + +:: + + Pagure Project Name + Jenkins Project Name + Jenkins Token + Jenkins Url + + All of which are required field. + +* The jenkins token is any string that you give here. The only thing that should be kept in mind that this token should be same through out. + +* This will give a POST URL which will be used for Job Notification in Jenkins + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. + +* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: All Event + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +How to install Pagure CI +======================== + +Pagure CI requires `fedmsg` to run since it uses a consumer to get messages and take appropriate actions. +The dependency that is required is `fedmdg-hubs`. For that the steps are given. +To install the dependencies required: + + `dnf install fedmsg-hub` + +`fedmsg` apart from the consumer require a file that tells to which cosumer it should listen to. This file basically enable the consumer in PagureCI/. For doing that, we need to place this file in appropriate directory. + + `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` + +Since the deployment is done using rpm, the next step is covered using `setup.py` which binds the consumer with the environment, this is done while building the rpm so if rpm is already built this is not explicitly required. + + `python setup.py install` + +Run the service: + + `sudo systemctl enable fedmsg-hub.service` + + `sudo systemctl start fedmsg-hub.service` + + + diff --git a/pagure/pagureCI/__init__.py b/pagure/pagureCI/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pagure/pagureCI/__init__.py +++ /dev/null diff --git a/pagure/pagureCI/consumer.py b/pagure/pagureCI/consumer.py deleted file mode 100644 index f45ad93..0000000 --- a/pagure/pagureCI/consumer.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -import fedmsg.consumers -from pagure.hooks import jenkins_hook -import pagure.lib -from pagure.lib import pagure_ci -from pagure.lib.model import BASE, Project, User -from pagure import APP, SESSION -PAGURE_MAIN_REPO = '{base}{name}.git' -PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' - - -class Integrator(fedmsg.consumers.FedmsgConsumer): - topic = [ - 'io.pagure.prod.pagure.pull-request.comment.added', - 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.prod.jenkins.build', - ] - - config_key = 'integrator.enabled' - - - def __init__(self, hub): - super(Integrator, self).__init__(hub) - - def consume(self, msg): - topic, msg = msg['topic'], msg['body'] - self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) - msg = msg['msg'] - try: - if topic.endswith('.pull-request.comment.added'): - if is_rebase(msg): - self.trigger_build(msg) - elif topic.endswith('.pull-request.new'): - self.trigger_build(msg) - else: - self.process_build(msg) - except jenkins_hook.ConfigNotFound as exc: - self.log.info('Unconfigured project %r', str(exc)) - - def trigger_build(self, msg): - pr_id = msg['pullrequest']['id'] - project = msg['pullrequest']['project']['name'] - branch = msg['pullrequest']['branch_from'] - - for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): - repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - self.log.info("Trigger on %s PR #%s from %s: %s", - project, pr_id, repo, branch) - - pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) - - def process_build(self, msg): - for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): - pagure_ci.process_build(self.log, cfg, msg['build']) - - -def get_repo(cfg, msg): - url = PAGURE_MAIN_REPO - if msg['pullrequest']['repo_from']['parent']: - url = PAGURE_FORK_REPO - return url.format( - base=APP.config['APP_URL'], - user=msg['pullrequest']['repo_from']['user']['name'], - name=msg['pullrequest']['repo_from']['name']) - - -def is_rebase(msg): - if msg['pullrequest']['status'] != 'Open': - return False - try: - print msg - return msg['pullrequest']['comments'][-1]['notification'] - except (IndexError, KeyError): - return False diff --git a/pagureCI/README.rst b/pagureCI/README.rst new file mode 100644 index 0000000..d79ab58 --- /dev/null +++ b/pagureCI/README.rst @@ -0,0 +1,107 @@ +Pagure CI +========= + +This is to setup Pagure CI for development. It is assumed that all the dependencies +are resolved. It is advised to use a virtual envivironment for development. + + * Run:: + + python setup.py develop + + +Now in pagureCI/consumer.py add the following elements in `topic` list + +:: + + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, after you login to your Jenkins Instance. + + +* Go to Manage Jenkins -> Configuire Global Security and under that select 'Project-based Matrix Authorization Strategy' + +* Add a user and give all the permission to that user. + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name you filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: All Event + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +* After all the configuration done, go to the dev instance of pagure running and under project settings in `Plugin` select Pagure CI and fill the appropriate information. Which on submiting should give you a POST url. + +* Copy and paste the URL in the Notification section under the Jenkins project you want the CI to work, + + +Get It Running: +=============== + +In one terminal window run: + +:: + + fedmsg-relay + +Another window: + +:: + + fedmsg-hub + +* Now clone the project locally and make a branch. make some changes and push it to the repo and try to make a PR. You will notice if everything works fine a lot of logs in the server console and `build fail` flag on the PR. Build fail because there is no git server running. diff --git a/pagureCI/__init__.py b/pagureCI/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagureCI/__init__.py diff --git a/pagureCI/consumer.py b/pagureCI/consumer.py new file mode 100644 index 0000000..9960416 --- /dev/null +++ b/pagureCI/consumer.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +import pagure.lib +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +from pagure import APP, SESSION +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=APP.config['APP_URL'], + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + print msg + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/setup.py b/setup.py index 544b80c..efcf7ab 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,6 @@ setup( install_requires=get_requirements(), entry_points=""" [moksha.consumer] - integrator = pagure.pagureCI.consumer:Integrator + integrator = pagureCI.consumer:Integrator """ )