From 1a38700f3d44dac1c82a22a01af1c64b8b05b036 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: Fix major issues, restructuring model and improved interaction --- diff --git a/pagure/consumer.py b/pagure/consumer.py deleted file mode 100644 index 5725240..0000000 --- a/pagure/consumer.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -import fedmsg.consumers -from pagure.hooks import jenkins_hook -from pagure.lib import pagure_ci -from pagure.lib.model import BASE, Project, User -PAGURE_MAIN_REPO = '{base}{name}.git' -PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' - - -class Integrator(fedmsg.consumers.FedmsgConsumer): - topic = [ - 'io.pagure.prod.pagure.pull-request.comment.added', - 'org.fedoraproject.dev.pagure.pull-request.new', - 'org.fedoraproject.dev.pagure.pull-request.comment.added', - 'io.pagure.prod.pagure.pull-request.new', - 'org.fedoraproject.prod.jenkins.build', - ] - - config_key = 'integrator.enabled' - - def __init__(self, hub): - super(Integrator, self).__init__(hub) - pagure_ci.connect_db() - - def consume(self, msg): - topic, msg = msg['topic'], msg['body'] - self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) - msg = msg['msg'] - try: - if topic.endswith('.pull-request.comment.added'): - if is_rebase(msg): - self.trigger_build(msg) - elif topic.endswith('.pull-request.new'): - self.trigger_build(msg) - else: - self.process_build(msg) - except jenkins_hook.ConfigNotFound as exc: - self.log.info('Unconfigured project %r', str(exc)) - - def trigger_build(self, msg): - pr_id = msg['pullrequest']['id'] - project = msg['pullrequest']['project']['name'] - branch = msg['pullrequest']['branch_from'] - - for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): - repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) - self.log.info("Trigger on %s PR #%s from %s: %s", - project, pr_id, repo, branch) - - pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) - - def process_build(self, msg): - for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): - pagure_ci.process_build(self.log, cfg, msg['build']) - - -def get_repo(cfg, msg): - url = PAGURE_MAIN_REPO - if msg['pullrequest']['repo_from']['parent']: - url = PAGURE_FORK_REPO - return url.format( - base=cfg.pagure_url, - user=msg['pullrequest']['repo_from']['user']['name'], - name=msg['pullrequest']['repo_from']['name']) - - -def is_rebase(msg): - if msg['pullrequest']['status'] != 'Open': - return False - try: - return msg['pullrequest']['comments'][-1]['notification'] - except (IndexError, KeyError): - return False diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index d22f05d..1d162ba 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -1,12 +1,5 @@ # -*- coding: utf-8 -*- -""" - (c) 2014 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - -""" import os import uuid @@ -15,13 +8,13 @@ import sqlalchemy as sa import pygit2 from wtforms import validators, TextField, BooleanField from flask.ext import wtf -from sqlalchemy.orm import relation -from sqlalchemy.orm import backref +from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base from pagure.hooks import BaseHook, RequiredIf from pagure.lib.model import BASE, Project from pagure import get_repo_path +from pagure import APP class PagureCI(BASE): @@ -31,7 +24,7 @@ class PagureCI(BASE): id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( sa.Integer, - sa.ForeignKey('projects.id', onupdate='CASCADE'), + sa.ForeignKey('projects.id', onupdate='CASCADE',ondelete='CASCADE'), nullable=False, unique=False, index=True) @@ -40,8 +33,7 @@ class PagureCI(BASE): display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) - pagure_url = sa.Column(sa.String(255), nullable=False, - default='https://pagure.io/') + pagure_token = sa.Column(sa.String(64)) jenkins_name = sa.Column(sa.String(255)) @@ -64,6 +56,7 @@ class PagureCI(BASE): def __init__(self): self.hook_token = uuid.uuid4().hex + self.display_name = 'Jenkins' def __repr__(self): return ''.format(self) @@ -96,17 +89,11 @@ class JenkinsForm(wtf.Form): name = TextField('Name', [validators.Required(), validators.Length(max=64)]) - display_name = TextField('Display name', - [validators.Required(), - validators.Length(max=64)], - default='Jenkins') + pagure_name = TextField('Name of project in Pagure', [validators.Required(), validators.Length(max=255)]) - pagure_url = TextField('Pagure URL', - [validators.Required(), - validators.Length(max=255)], - default='https://pagure.io/') + pagure_token = TextField('Pagure token', [validators.Required()]) @@ -122,7 +109,7 @@ class JenkinsForm(wtf.Form): active = BooleanField('Active', [validators.Optional()]) -class Hook(BaseHook): +class PagureCiHook(BaseHook): ''' Jenkins hooks. ''' name = 'Pagure CI' @@ -130,13 +117,20 @@ class Hook(BaseHook): ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI - backref = 'pagure_ci_hook' + backref = 'hook_pagure_ci' form_fields = [ - 'display_name', 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'name', 'pagure_name', 'pagure_token', 'jenkins_name', 'jenkins_url', 'jenkins_token', 'active' ] @classmethod + def set_up(cls, project): + ''' Install the generic post-receive hook that allow us to call + multiple post-receive hooks as set per plugin. + ''' + pass + + @classmethod def install(cls, project, dbobj): ''' Method called to install the hook for a project. @@ -144,22 +138,7 @@ class Hook(BaseHook): should be installed ''' - repopath = get_repo_path(project) - - hook_files = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'files') - repo_obj = pygit2.Repository(repopath) - - # Configure the hook - # repo_obj.config.set_multivar(dbobj) - - # Install the hook itself - hook_file = os.path.join(hook_files, 'jenkins_hook.py') - if not os.path.exists(hook_file): - os.symlink( - hook_file, - os.path.join(repopath, 'hooks', 'jenkins_hook.py') - ) + pass @classmethod def remove(cls, project): @@ -169,8 +148,4 @@ class Hook(BaseHook): should be installed ''' - repopath = get_repo_path(project) - - #hook_path = os.path.join(repopath, 'hooks', 'post-receive.irc') - # if os.path.exists(hook_path): - # os.unlink(hook_path) + pass diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py index 8157c5f..ec5bad3 100644 --- a/pagure/lib/pagure_ci.py +++ b/pagure/lib/pagure_ci.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy import create_engine from pagure.hooks import jenkins_hook from pagure.lib import model -from pagure import APP +from pagure import APP, SESSION import json import logging @@ -13,9 +13,6 @@ import logging import requests import jenkins -os.environ.setdefault('INTEGRATOR_SETTINGS', '/etc/poormanci.conf') - - APP.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) APP.logger.setLevel(logging.INFO) @@ -23,21 +20,10 @@ PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' -db_session = None - -def connect_db(): - global db_session - engine = create_engine(APP.config['DB_URL'], convert_unicode=True) - db_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) - model.BASE.query = db_session.query_property() - - - def process_pr(logger, cfg, pr_id, repo, branch): post_data(logger, - JENKINS_TRIGGER_URL.format(base=cfg.jenkins_url, project=cfg.jenkins_name), + JENKINS_TRIGGER_URL.format( + base=cfg.jenkins_url, project=cfg.jenkins_name), {'token': cfg.jenkins_token, 'cause': pr_id, 'REPO': repo, @@ -67,7 +53,7 @@ def process_build(logger, cfg, build_id): # Comment in Pagure logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) try: - post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, + post_flag(logger, cfg.display_name, APP.config['APP_URL'], cfg.pagure_token, cfg.pagure_name, pr_id, result, url) except KeyError as exc: logger.warning('Unknown build status', exc_info=exc) @@ -92,35 +78,5 @@ def post_data(logger, *args, **kwargs): resp = requests.post(*args, **kwargs) logger.debug('Received response status %s', resp.status_code) if resp.status_code < 200 or resp.status_code >= 300: - logger.error('Network request failed: %d: %s', resp.status_code, resp.text) - - -@APP.route('/hooks//build-finished', methods=['POST']) -def hook_finished(token): - try: - data = json.loads(flask.request.get_data()) - cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] - build_id = data['build']['number'] - if token != cfg.hook_token: - raise ValueError('Token mismatch') - except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: - APP.logger.error('Error processing jenkins notification', exc_info=exc) - return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) - APP.logger.info('Received jenkins notification') - process_build(APP.logger, cfg, build_id) - return ('', 204) - -def cleanup_url(url): - """Make sure there is trailing slash.""" - return url.rstrip('/') + '/' - - -@APP.before_request -def before_request(): - if db_session is None: - connect_db() - - -@APP.teardown_appcontext -def shutdown_session(exception=None): - db_session.remove() + logger.error('Network request failed: %d: %s', + resp.status_code, resp.text) diff --git a/pagure/pagureCI/__init__.py b/pagure/pagureCI/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pagure/pagureCI/__init__.py diff --git a/pagure/pagureCI/consumer.py b/pagure/pagureCI/consumer.py new file mode 100644 index 0000000..4ef85de --- /dev/null +++ b/pagure/pagureCI/consumer.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +import pagure.lib +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +from pagure import APP +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + SESSION = None + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + SESSION = pagure.lib.create_session(APP.config['DB_URL']) + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=APP.config['APP_URL'], + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + print msg + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 0de4a79..9f3033d 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -9,7 +9,6 @@ """ import flask -import uuid from sqlalchemy.exc import SQLAlchemyError from straight.plugin import load @@ -22,6 +21,10 @@ from pagure import APP, SESSION, login_required, is_repo_admin from pagure.lib.model import BASE from pagure.exceptions import FileNotFoundException from pagure.hooks.jenkins_hook import PagureCI +from pagure.hooks import jenkins_hook +from pagure.lib import model, pagure_ci + +import json # pylint: disable=E1101 @@ -90,18 +93,16 @@ def view_plugin(repo, plugin, username=None, full=True): post_token = None dbobj = plugin.db_object() - post_token_obj = BASE.metadata.bind.query(PagureCI).filter( - PagureCI.pagure_name == repo.name).first() - - if hasattr(post_token_obj, 'hook_token'): - post_token = getattr(post_token_obj, 'hook_token') - if hasattr(repo, plugin.backref): dbobj = getattr(repo, plugin.backref) + # There should always be only one, but let's double check if dbobj and len(dbobj) > 0: dbobj = dbobj[0] new = False + # hook_token of pagure shouldn't leak so to put a check on it + if hasattr(dbobj, "hook_token") and plugin.backref == "hook_pagure_ci": + post_token = dbobj.hook_token else: dbobj = plugin.db_object() @@ -169,3 +170,20 @@ def view_plugin(repo, plugin, username=None, full=True): form=form, post_token=post_token, fields=fields) + + +@APP.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs( + data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, jenkins_hook.ConfigNotFound) as exc: + APP.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + APP.logger.info('Received jenkins notification') + pagure_ci.process_build(APP.logger, cfg, build_id) + return ('', 204) diff --git a/setup.py b/setup.py index 02d354b..544b80c 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,6 @@ setup( install_requires=get_requirements(), entry_points=""" [moksha.consumer] - integrator = pagure.consumer:Integrator + integrator = pagure.pagureCI.consumer:Integrator """ )