diff --git a/.gitignore b/.gitignore index 2519925..d99a465 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ dev/docker/test_env # A possible symlink for the testsuite /repospanner +/repohookrunner diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index c19e9d5..a1a9c1d 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -226,6 +226,7 @@ REPOSPANNER_ADMIN_MIGRATION = False # Example entry: # 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', # 'repo_prefix': 'pagure/', +# 'hook': None, # 'ca': '', # 'admin_cert': {'cert': '', # 'key': ''}, diff --git a/pagure/cli/admin.py b/pagure/cli/admin.py index 088d190..2e394af 100644 --- a/pagure/cli/admin.py +++ b/pagure/cli/admin.py @@ -14,6 +14,8 @@ import argparse import datetime import logging import os +import requests +from string import Template import sys import arrow @@ -340,6 +342,37 @@ def _parser_block_user(subparser): local_parser.set_defaults(func=do_block_user) +def _parser_upload_repospanner_hooks(subparser): + """ Set up the CLI argument parser to upload repospanner hook. + + Args: + subparser: An argparse subparser + """ + local_parser = subparser.add_parser( + "upload-repospanner-hook", help="Upload repoSpanner hook script" + ) + local_parser.add_argument( + "region", help="repoSpanner region where to " "upload hook" + ) + local_parser.set_defaults(func=do_upload_repospanner_hooks) + + +def _parser_ensure_project_hooks(subparser): + """ Set up the CLI argument parser to ensure project hooks are setup + + Args: + subparser: An argparse subparser + """ + local_parser = subparser.add_parser( + "ensure-project-hooks", + help="Ensure all projects have their hooks setup", + ) + local_parser.add_argument( + "hook", help="repoSpanner hook ID to set", default=None + ) + local_parser.set_defaults(func=do_ensure_project_hooks) + + def parse_arguments(args=None): """ Set-up the argument parsing. """ parser = argparse.ArgumentParser( @@ -386,6 +419,12 @@ def parse_arguments(args=None): # block-user _parser_block_user(subparser) + # upload-repospanner-hooks + _parser_upload_repospanner_hooks(subparser) + + # ensure-project-hooks + _parser_ensure_project_hooks(subparser) + return parser.parse_args(args) @@ -867,6 +906,74 @@ def do_block_user(args): session.commit() +def do_upload_repospanner_hooks(args): + """ Upload hooks to repoSpanner + + Args: + args (argparse.Namespace): Parsed arguments + """ + regioninfo = pagure.config.config["REPOSPANNER_REGIONS"].get(args.region) + if not regioninfo: + raise ValueError( + "repoSpanner region %s not in config file" % args.region + ) + + env = { + "config": os.environ.get("PAGURE_CONFIG", "/etc/pagure/pagure.cfg"), + "pypath": os.environ.get("PYTHONPATH", "None"), + } + sourcefile = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "../hooks/files/repospannerhook", + ) + ) + with open(sourcefile, "r") as source: + template = source.read() + hookcontents = Template(template).substitute(env) + + resp = requests.post( + "%s/admin/hook/admin.git/upload" % regioninfo["url"], + data=hookcontents, + headers={"X-Object-Size": str(len(hookcontents))}, + verify=regioninfo["ca"], + cert=( + regioninfo["admin_cert"]["cert"], + regioninfo["admin_cert"]["key"], + ), + ) + resp.raise_for_status() + resp = resp.json() + _log.debug("Response json: %s", resp) + if not resp["Success"]: + raise Exception("Error in repoSpanner API call: %s" % resp["Error"]) + hook = resp["Info"] + if hook == regioninfo["hook"]: + print("Hook was up-to-date") + else: + print("Hook ID for region %s: %s" % (args.region, hook)) + return hook + + +def do_ensure_project_hooks(args): + """ Ensures that all projects have their hooks setup + + Args: + args (argparse.Namespace): Parsed arguments + """ + projects = [] + query = session.query(pagure.lib.model.Project).order_by( + pagure.lib.model.Project.id + ) + for project in query.all(): + print("Ensuring hooks for %s" % project.fullname) + projects.append(project.fullname) + pagure.lib.git.set_up_project_hooks( + project, project.repospanner_region, hook=args.hook + ) + return projects + + def main(): """ Start of the application. """ diff --git a/pagure/default_config.py b/pagure/default_config.py index 3a8d44f..64d2c7f 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -495,6 +495,7 @@ REPOSPANNER_ADMIN_MIGRATION = False # Example entry: # 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', # 'repo_prefix': 'pagure/', +# 'hook': None, # 'ca': '', # 'admin_cert': {'cert': '', # 'key': ''}, diff --git a/pagure/hooks/__init__.py b/pagure/hooks/__init__.py index d5467fc..9a17e80 100644 --- a/pagure/hooks/__init__.py +++ b/pagure/hooks/__init__.py @@ -141,15 +141,12 @@ class BaseHook(object): # as the hook script will be arranged by repo creation. return - repopaths = [ - project.repopath(repotype) for repotype in pagure.lib.REPOTYPES - ] - hook_files = os.path.join( os.path.dirname(os.path.realpath(__file__)), "files" ) - for repopath in repopaths: + for repotype in pagure.lib.REPOTYPES: + repopath = project.repopath(repotype) if repopath is None: continue @@ -413,23 +410,33 @@ def run_project_hooks( raise SystemExit(1) -def run_hook_file(hooktype): - """ Runs a specific hook by grabbing the changes and running functions. +def extract_changes(from_stdin): + """ Extracts a changes dict from either stdin or argv Args: - hooktype (string): The name of the hook to run: pre-receive, update - or post-receive + from_stdin (bool): Whether to use stdin. If false, uses argv """ changes = {} - if hooktype in ("pre-receive", "post-receive"): + if from_stdin: for line in sys.stdin: (oldrev, newrev, refname) = line.strip().split(" ", 2) changes[refname] = (oldrev, newrev) - elif hooktype == "update": + else: (refname, oldrev, newrev) = sys.argv[1:] changes[refname] = (oldrev, newrev) - else: + return changes + + +def run_hook_file(hooktype): + """ Runs a specific hook by grabbing the changes and running functions. + + Args: + hooktype (string): The name of the hook to run: pre-receive, update + or post-receive + """ + if hooktype not in ("pre-receive", "update", "post-receive"): raise ValueError("Hook type %s not valid" % hooktype) + changes = extract_changes(from_stdin=hooktype != "update") session = pagure.lib.create_session(pagure_config["DB_URL"]) if not session: diff --git a/pagure/hooks/files/repospannerhook b/pagure/hooks/files/repospannerhook new file mode 100755 index 0000000..2669cda --- /dev/null +++ b/pagure/hooks/files/repospannerhook @@ -0,0 +1,68 @@ +#!/bin/env python3 +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + +""" + +import os +import sys + +# These fields get filled in by upload-repospanner-hooks +os.environ['PAGURE_CONFIG'] = '${config}' +PYPATH = "${pypath}" + +# Prepare code imports +if PYPATH: + sys.path.append(PYPATH) + +import pagure +import pagure.lib +from pagure.hooks import run_project_hooks, extract_changes +from pagure.config import config as pagure_config + + +# Get information from the environment +hooktype = os.environ.get('HOOKTYPE') + +is_internal = os.environ.get('extra_internal', False) == 'yes' +pushuser = os.environ['extra_username'] +repotype = os.environ['extra_repotype'] +project_name = os.environ['extra_project_name'] +project_user = os.environ.get('extra_project_user', None) +project_namespace = os.environ.get('extra_project_namespace', None) +pruid = os.environ.get('extra_pull_request_uid', None) + +changes = extract_changes(from_stdin=hooktype != 'update') + +session = pagure.lib.create_session(pagure_config["DB_URL"]) +if not session: + raise Exception("Unable to initialize db session") + +gitdir = os.path.abspath(os.environ["GIT_DIR"]) + +project = pagure.lib._get_project( + session, project_name, project_user, project_namespace +) + +pull_request = None +if pruid: + pull_request = pagure.lib.get_request_by_uid( + session, os.environ["pull_request_uid"] + ) + +run_project_hooks( + session, + pushuser, + project, + hooktype, + repotype, + gitdir, + changes, + is_internal, + pull_request +) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 50b8c1d..aa9121c 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -41,6 +41,7 @@ from pagure.config import config as pagure_config from pagure.lib import model from pagure.lib.repo import PagureRepo from pagure.lib import tasks +import pagure.hooks # from pagure.hooks import run_project_hooks @@ -957,6 +958,18 @@ class TemporaryClone(object): regioninfo = pagure_config["REPOSPANNER_REGIONS"][ self._project.repospanner_region ] + + extra.update( + { + "username": username, + "repotype": self._repotype, + "project_name": self._project.name, + "project_user": self._project.user.username + if self._project.is_fork + else "", + "project_namespace": self._project.namespace or "", + } + ) opts = [ "-c", "http.sslcainfo=%s" % regioninfo["ca"], @@ -964,8 +977,6 @@ class TemporaryClone(object): "http.sslcert=%s" % regioninfo["push_cert"]["cert"], "-c", "http.sslkey=%s" % regioninfo["push_cert"]["key"], - "-c", - "http.extraHeader=X-Extra-Username: %s" % username, ] for extrakey in extra: val = extra[extrakey] @@ -978,6 +989,7 @@ class TemporaryClone(object): "Running a git push of %s to %s" % (pushref, self._project.fullname) ) + _log.debug("Opts: %s", opts) env = os.environ.copy() env["GL_USER"] = username env.update(extra) @@ -993,6 +1005,7 @@ class TemporaryClone(object): # this way, we can be sure to get the output logged remotes = [] for line in ex.output.decode("utf-8").split("\n"): + _log.info("Remote line: %s", line) if line.startswith("remote: "): _log.debug("Remote: %s" % line) remotes.append(line[len("remote: ") :].strip()) @@ -2148,6 +2161,56 @@ def delete_project_repos(project): ) +def set_up_project_hooks(project, region, hook=None): + """ Makes sure the git repositories for a project have their hooks setup. + + Args: + project (model.Project): Project to set up hooks for + region (string or None): repoSpanner region to set hooks up for + hook (string): The hook ID to set up in repoSpanner (tests only) + """ + if region is None: + # This repo is not on repoSpanner, create hooks locally + pagure.hooks.BaseHook.set_up(project) + else: + regioninfo = pagure_config["REPOSPANNER_REGIONS"].get(region) + if not regioninfo: + raise ValueError( + "Invalid repoSpanner region %s looked up" % region + ) + if not hook: + hook = regioninfo["hook"] + if not hook: + # No hooks to set up for this region + return + + for repotype in pagure.lib.REPOTYPES: + data = { + "Reponame": project._repospanner_repo_name(repotype, region), + "UpdateRequest": { + "hook-prereceive": hook, + "hook-update": hook, + "hook-postreceive": hook, + }, + } + resp = requests.post( + "%s/admin/editrepo" % regioninfo["url"], + json=data, + verify=regioninfo["ca"], + cert=( + regioninfo["admin_cert"]["cert"], + regioninfo["admin_cert"]["key"], + ), + ) + resp.raise_for_status() + resp = resp.json() + _log.debug("Response json: %s", resp) + if not resp["Success"]: + raise Exception( + "Error in repoSpanner API call: %s" % resp["Error"] + ) + + def _create_project_repo(project, region, templ, ignore_existing, repotype): """ Creates a single specific git repository on disk or repoSpanner @@ -2242,3 +2305,5 @@ def create_project_repos(project, region, templ, ignore_existing): for created in created_dirs: shutil.rmtree(created) raise + + set_up_project_hooks(project, region) diff --git a/tests/__init__.py b/tests/__init__.py index aeb0d2e..ba00f40 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -106,6 +106,7 @@ REPOSPANNER_ADMIN_MIGRATION = %(repospanner_admin_migration)s REPOSPANNER_REGIONS = { 'default': {'url': 'https://nodea.regiona.repospanner.local:%(repospanner_gitport)s', 'repo_prefix': 'pagure/', + 'hook': None, 'ca': '%(path)s/repospanner/pki/ca.crt', 'admin_cert': {'cert': '%(path)s/repospanner/pki/admin.crt', 'key': '%(path)s/repospanner/pki/admin.key'}, diff --git a/tests/test_pagure_lib_git_auth.py b/tests/test_pagure_lib_git_auth.py index d6e0b6a..3f67d8a 100644 --- a/tests/test_pagure_lib_git_auth.py +++ b/tests/test_pagure_lib_git_auth.py @@ -93,19 +93,13 @@ class PagureLibGitAuthtests(tests.Modeltests): tests.add_content_git_repo( os.path.join(self.path, 'repos', 'hooktest.git')) - output = self.app.get('/hooktest/edit/master/f/sources') - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - data = { 'content': 'foo\n bar\n baz', 'commit_title': 'test commit', 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'master', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( @@ -138,12 +132,6 @@ class PagureLibGitAuthtests(tests.Modeltests): tests.add_content_git_repo( os.path.join(self.path, 'repos', 'hooktest.git')) - output = self.app.get('/hooktest/edit/master/f/sources') - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - # Try editing master branch, should fail (only PRs allowed) data = { 'content': 'foo\n bar\n baz', @@ -151,7 +139,7 @@ class PagureLibGitAuthtests(tests.Modeltests): 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'master', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( '/hooktest/edit/master/f/sources', data=data, @@ -172,7 +160,7 @@ class PagureLibGitAuthtests(tests.Modeltests): 'commit_message': 'Online commits from the gure.lib.get', 'email': 'bar@pingou.com', 'branch': 'source', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( diff --git a/tests/test_pagure_repospanner.py b/tests/test_pagure_repospanner.py index 91af881..81ac68a 100644 --- a/tests/test_pagure_repospanner.py +++ b/tests/test_pagure_repospanner.py @@ -14,6 +14,7 @@ __requires__ = ['SQLAlchemy >= 0.8'] import pkg_resources import datetime +import munch import unittest import shutil import subprocess @@ -34,6 +35,7 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import pagure.lib +import pagure.cli.admin import tests @@ -80,11 +82,20 @@ hooks: hostname: myhostname bind: ro_bind: - /usr: /usr + - - /usr + - /usr + - - %(codepath)s + - %(codepath)s + - - %(path)s + - %(path)s + - - %(crosspath)s + - %(crosspath)s symlink: - usr/lib64: /lib64 - usr/bin: /bin - runner: /usr/bin/repohookrunner + - - usr/lib64 + - /lib64 + - - usr/bin + - /bin + runner: %(hookrunner_bin)s user: 0 """ @@ -123,6 +134,16 @@ class PagureRepoSpannerTests(tests.Modeltests): if not self.repospanner_binary: raise unittest.SkipTest('repoSpanner not found') + hookrunbin = os.path.join(os.path.dirname(self.repospanner_binary), + 'repohookrunner') + if not os.path.exists(hookrunbin): + raise Exception('repoSpanner found, but repohookrunner not') + + codepath = os.path.normpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../")) + # Only run the setUp() function if we are actually going ahead and run # this test. The reason being that otherwise, setUp will set up a # database, but because we "error out" from setUp, the tearDown() @@ -132,8 +153,11 @@ class PagureRepoSpannerTests(tests.Modeltests): # TODO: Find free ports configvals = { 'path': self.path, + 'crosspath': tests.tests_state["path"], 'gitport': 8443, 'rpcport': 8444, + 'codepath': codepath, + 'hookrunner_bin': hookrunbin, } os.mkdir(os.path.join(self.path, 'repospanner')) @@ -216,7 +240,8 @@ class PagureRepoSpannerTests(tests.Modeltests): class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): config_values = { - 'repospanner_new_repo': "'default'" + 'repospanner_new_repo': "'default'", + 'authbackend': 'test_auth', } @patch('pagure.ui.app.admin_session_timedout') @@ -232,14 +257,11 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): self.assertIn( 'Create new Project', output_text) - csrf_token = output_text.split( - 'name="csrf_token" type="hidden" value="')[1].split('">')[0] - data = { 'name': 'project-1', 'description': 'Project #1', 'create_readme': 'y', - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post('/new/', data=data, follow_redirects=True) @@ -258,9 +280,8 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): output.get_data(as_text=True)) with tests.user_set(self.app.application, tests.FakeUser(username='pingou')): - csrf_token = self.get_csrf() data = { - 'csrf_token': csrf_token, + 'csrf_token': self.get_csrf(), } output = self.app.post( @@ -284,6 +305,113 @@ class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): repodirlist = os.listdir(os.path.join(self.path, 'repos')) self.assertEqual(repodirlist, ['pseudo']) + @patch('pagure.ui.app.admin_session_timedout') + def test_hooks(self, ast): + """ Test hook setting and running works. """ + ast.return_value = False + pagure.cli.admin.session = self.session + + # Upload the hook script to repoSpanner + args = munch.Munch({'region': 'default'}) + hookid = pagure.cli.admin.do_upload_repospanner_hooks(args) + + user = tests.FakeUser(username='foo') + with tests.user_set(self.app.application, user): + data = { + 'name': 'project-1', + 'description': 'Project #1', + 'create_readme': 'y', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post('/new/', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + '
\nProject #1', + output_text) + self.assertIn( + 'Overview - project-1 - Pagure', output_text) + self.assertIn('Added the README', output_text) + + output = self.app.get('/project-1/settings') + self.assertIn( + 'This repository is on repoSpanner region default', + output.get_data(as_text=True)) + + # Check file before the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, '# project-1\n\nProject #1') + + # Set the hook + args = munch.Munch({'hook': hookid}) + projects = pagure.cli.admin.do_ensure_project_hooks(args) + self.assertEqual(["project-1"], projects) + + with tests.user_set(self.app.application, user): + # Set editing Denied + self.set_auth_status(False) + + # Try to make an edit in the repo + data = { + 'content': 'foo\n bar\n baz', + 'commit_title': 'test commit', + 'commit_message': 'Online commit', + 'email': 'foo@bar.com', + 'branch': 'master', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post( + '/project-1/edit/master/f/README.md', data=data, + follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + "Remote hook declined the push: ", + output_text + ) + self.assertIn( + "Denied push for ref 'refs/heads/master' for user 'foo'\n" + "All changes have been rejected", + output_text + ) + + # Check file after the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, '# project-1\n\nProject #1') + + # Set editing Allowed + self.set_auth_status(True) + + # Try to make an edit in the repo + data = { + 'content': 'foo\n bar\n baz', + 'commit_title': 'test commit', + 'commit_message': 'Online commit', + 'email': 'foo@bar.com', + 'branch': 'master', + 'csrf_token': self.get_csrf(), + } + + output = self.app.post( + '/project-1/edit/master/f/README.md', data=data, + follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Commits - project-1 - Pagure', output_text) + + # Check file after the commit: + output = self.app.get('/project-1/raw/master/f/README.md') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertEqual(output_text, 'foo\n bar\n baz') + if __name__ == '__main__': unittest.main(verbosity=2)