diff --git a/doc/configuration.rst b/doc/configuration.rst index a2acff8..2fa4246 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -206,9 +206,16 @@ Configure Gitolite Pagure uses `gitolite `_ as an authorization layer. Gitolite relies on `SSH `_ for -the authentication. In other words, SSH lets you in and gitolite checks if you -are allowed to do what you are trying to do once you are inside. +the authentication. In other words, SSH lets you in and gitolite checks if +you are allowed to do what you are trying to do once you are inside. +Pagure supports both gitolite 2 and gitolite 3 and the code generating +the gitolite configuration can be customized for easier integration with +other systems (cf :ref:`custom-gitolite`). + + +**gitolite 2 and 3** +~~~~~~~~~~~~~~~~~~~~ GITOLITE_HOME ~~~~~~~~~~~~~ @@ -217,15 +224,6 @@ This configuration key points to the home directory of the user under which gitolite is ran. -GITOLITE_VERSION -~~~~~~~~~~~~~~~~ - -This configuration key specifies which version of gitolite you are -using, it can be either ``2`` or ``3``. - -Defaults to: ``3``. - - GITOLITE_KEYDIR ~~~~~~~~~~~~~~~ @@ -243,6 +241,26 @@ This configuration key points to the gitolite.conf file where pagure writes the gitolite repository access configuration. +GITOLITE_BACKEND +~~~~~~~~~~~~~~~~ + +This configuration key allows specifying which helper method to use to +generate and compile gitolite's configuration file. + +By default pagure provides the following backends: + +- `test_auth`: simple debugging backend printing and returning the string ``Called GitAuthTestHelper.generate_acls()`` +- `gitolite2`: allows deploying pagure on the top of gitolite 2 +- `gitolite3`: allows deploying pagure on the top of gitolite 3 + +Defaults to: ``gitolite3`` + +.. note:: These options can be expended, cf :ref:`custom-gitolite`. + + +**gitolite 2 only** +~~~~~~~~~~~~~~~~~~~ + GL_RC ~~~~~ @@ -778,3 +796,14 @@ UPLOAD_FOLDER This configuration key used to be use to specify where the uploaded releases are available. It has been replaced by `UPLOAD_FOLDER_PATH` in the release 2.10 of pagure. + + +GITOLITE_VERSION +~~~~~~~~~~~~~~~~ + +This configuration key specifies which version of gitolite you are +using, it can be either ``2`` or ``3``. + +Defaults to: ``3``. + +This has been replaced by `GITOLITE_BACKEND` in the release 3.0 of pagure. diff --git a/doc/custom_gitolite_conf.rst b/doc/custom_gitolite_conf.rst new file mode 100644 index 0000000..6f03f20 --- /dev/null +++ b/doc/custom_gitolite_conf.rst @@ -0,0 +1,36 @@ +.. _custom-gitolite: + +Customize the gitolite configuration +==================================== + +Pagure provides a mechanism to allow customizing the creation and +compilation of the configuration file of gitolite. + +To customize the gitolite configuration file, we invite you to look at the +`sources of the module pagure.lib.git_auth +`_. + +As you can see it defines the following class:: + + class GitAuthHelper(object): + + __metaclass__ = abc.ABCMeta + + @staticmethod + @abc.abstractmethod + def generate_acls(): + pass + +This will be the class you will have to inherit from in order to inject your +own code. +You will then declare an entry point in your `setup.py` following this +template:: + + entry_points=""" + [pagure.git_auth.helpers] + my_git_auth = my_pagure.my_module:MyGitAuthTestHelper + """ + +Then you can adjust pagure's configuration file to say:: + + GITOLITE_BACKEND = 'my_git_auth' diff --git a/doc/index.rst b/doc/index.rst index d283886..d3a6dbc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -39,6 +39,7 @@ Contents: install_pagure_loadjson install_pagure_logcom configuration + custom_gitolite_conf development contributing contributors diff --git a/pagure/default_config.py b/pagure/default_config.py index a808189..3d45208 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -172,6 +172,9 @@ GITOLITE_VERSION = 3 # Folder containing all the public ssh keys for gitolite GITOLITE_KEYDIR = None +# Backend to use to write down the gitolite configuration file +GITOLITE_BACKEND = 'gitolite3' + # Path to the gitolite.rc file GL_RC = None # Path to the /bin directory where the gitolite tools can be found diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 7f0a207..be31716 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -24,7 +24,6 @@ import tempfile import arrow import pygit2 -import werkzeug from sqlalchemy.exc import SQLAlchemyError from pygit2.remote import RemoteCollection @@ -89,157 +88,10 @@ Subject: {subject} return patch -def __read_file(filename): - """ Reads the specified file and return its content. - Returns None if it could not read the file for any reason. - """ - if not os.path.exists(filename): - _log.info('Could not find file: %s', filename) - else: - with open(filename) as stream: - return stream.read() - - -def write_gitolite_acls(session, configfile, preconf=None, postconf=None): - ''' Generate the configuration file for gitolite for all projects - on the forge. - ''' - _log.info('Write down the gitolite configuration file') - - preconfig = None - if preconf: - _log.info( - 'Loading the file to include at the top of the generated one') - preconfig = __read_file(preconf) - - postconfig = None - if postconf: - _log.info( - 'Loading the file to include at the end of the generated one') - postconfig = __read_file(postconf) - - global_pr_only = pagure.APP.config.get('PR_ONLY', False) - config = [] - groups = {} - query = session.query( - model.Project - ).order_by( - model.Project.id - ) - for project in query.all(): - _log.debug(' Processing project: %s', project.fullname) - for group in project.committer_groups: - if group.group_name not in groups: - groups[group.group_name] = [ - user.username for user in group.users] - - # Check if the project or the pagure instance enforce the PR only - # development model. - pr_only = project.settings.get('pull_request_access_only', False) - - for repos in ['repos', 'docs/', 'tickets/', 'requests/']: - if repos == 'repos': - # Do not grant access to project enforcing the PR model - if pr_only or (global_pr_only and not project.is_fork): - continue - repos = '' - - config.append('repo %s%s' % (repos, project.fullname)) - if repos not in ['tickets/', 'requests/']: - config.append(' R = @all') - if project.committer_groups: - config.append(' RW+ = @%s' % ' @'.join([ - group.group_name for group in project.committer_groups])) - config.append(' RW+ = %s' % project.user.user) - for user in project.committers: - if user != project.user: - config.append(' RW+ = %s' % user.user) - for deploykey in project.deploykeys: - access = 'R' - if deploykey.pushaccess: - access = 'RW+' - # Note: the replace of / with _ is because gitolite users can't - # contain a /. At first, this might look like deploy keys in a - # project called $namespace_$project would give access to the - # repos of a project $namespace/$project or vica versa, however - # this is NOT the case because we add the deploykey.id to the - # end of the deploykey name, which means it is unique. The - # project name is solely there to make it easier to determine - # what project created the deploykey for admins. - config.append(' %s = deploykey_%s_%s' % - (access, - werkzeug.secure_filename(project.fullname), - deploykey.id)) - config.append('') - - with open(configfile, 'w') as stream: - if preconfig: - stream.write(preconfig + '\n') - - for key, users in groups.iteritems(): - stream.write('@%s = %s\n' % (key, ' '.join(users))) - stream.write('\n') - - for row in config: - stream.write(row + '\n') - - if postconfig: - stream.write(postconfig + '\n') - - -def _get_gitolite_command(): - """ Return the gitolite command to run based on the info in the - configuration file. - """ - _log.info('Compiling the gitolite configuration') - gitolite_folder = pagure.APP.config.get('GITOLITE_HOME', None) - gitolite_version = pagure.APP.config.get('GITOLITE_VERSION', 3) - if gitolite_folder: - if gitolite_version == 2: - cmd = 'GL_RC=%s GL_BINDIR=%s gl-compile-conf' % ( - pagure.APP.config.get('GL_RC'), - pagure.APP.config.get('GL_BINDIR') - ) - elif gitolite_version == 3: - cmd = 'HOME=%s gitolite compile && HOME=%s gitolite trigger '\ - 'POST_COMPILE' % ( - pagure.APP.config.get('GITOLITE_HOME'), - pagure.APP.config.get('GITOLITE_HOME') - ) - else: - raise pagure.exceptions.PagureException( - 'Non-supported gitolite version "%s"' % gitolite_version - ) - _log.debug('Command: %s', cmd) - return cmd - - def generate_gitolite_acls(): tasks.generate_gitolite_acls.delay() -def _generate_gitolite_acls(): - """ Generate the gitolite configuration file for all repos - """ - _log.info('Refresh gitolite configuration') - pagure.lib.git.write_gitolite_acls( - pagure.SESSION, - pagure.APP.config['GITOLITE_CONFIG'], - preconf=pagure.APP.config.get('GITOLITE_PRE_CONFIG') or None, - postconf=pagure.APP.config.get('GITOLITE_POST_CONFIG') or None - ) - - cmd = _get_gitolite_command() - if cmd: - subprocess.Popen( - cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=pagure.APP.config['GITOLITE_HOME'] - ) - - def update_git(obj, repo, repofolder): """ Schedules an update_repo task after determining arguments. """ ticketuid = None diff --git a/pagure/lib/git_auth.py b/pagure/lib/git_auth.py new file mode 100644 index 0000000..1924962 --- /dev/null +++ b/pagure/lib/git_auth.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2015-2017 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" +from __future__ import print_function + +import abc +import logging +import os +import pkg_resources +import subprocess + +import werkzeug + +import pagure +import pagure.exceptions +from pagure import APP +from pagure.lib import model + +logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) +_log = logging.getLogger(__name__) + + +def get_git_auth_helper(backend, *args, **kwargs): + """ Instantiate and return the appropriate git auth helper backend. + + :arg backend: The name of the backend to find on the system (declared via + the entry_points in setup.py). + Pagure comes by default with the following backends: + test_auth, gitolite2, gitolite3 + :type backend: str + + """ + points = pkg_resources.iter_entry_points('pagure.git_auth.helpers') + classes = dict([(point.name, point.load()) for point in points]) + _log.debug("Found the following installed helpers %r" % classes) + cls = classes[backend] + _log.debug("Instantiating helper %r from backend key %r" % (cls, backend)) + return cls(*args, **kwargs) + + +class GitAuthHelper(object): + """ The class to inherit from when creating your own git authentication + helper. + """ + + __metaclass__ = abc.ABCMeta + + @classmethod + @abc.abstractmethod + def generate_acls(self): + """ This is the method that is called by pagure to generate the + configuration file. + """ + pass + + +def _read_file(filename): + """ Reads the specified file and return its content. + Returns None if it could not read the file for any reason. + """ + if not os.path.exists(filename): + _log.info('Could not find file: %s', filename) + else: + with open(filename) as stream: + return stream.read() + + +class Gitolite2Auth(GitAuthHelper): + """ A gitolite 2 authentication module. """ + + @classmethod + def write_gitolite_acls( + cls, session, configfile, preconf=None, postconf=None): + ''' Generate the configuration file for gitolite for all projects + on the forge. + + :arg cls: the current class + :type: Gitolite2Auth + :arg session: a session to connect to the database with + :arg configfile: the name of the configuration file to generate/write + :type configfile: str + :kwarg preconf: a file to include at the top of the configuration + file + :type preconf: None or str + :kwarg postconf: a file to include at the bottom of the + configuration file + :type postconf: None or str + + ''' + _log.info('Write down the gitolite configuration file') + + preconfig = None + if preconf: + _log.info( + 'Loading the file to include at the top of the generated one') + preconfig = _read_file(preconf) + + postconfig = None + if postconf: + _log.info( + 'Loading the file to include at the end of the generated one') + postconfig = _read_file(postconf) + + global_pr_only = pagure.APP.config.get('PR_ONLY', False) + config = [] + groups = {} + query = session.query( + model.Project + ).order_by( + model.Project.id + ) + for project in query.all(): + _log.debug(' Processing project: %s', project.fullname) + for group in project.committer_groups: + if group.group_name not in groups: + groups[group.group_name] = [ + user.username for user in group.users] + + # Check if the project or the pagure instance enforce the PR only + # development model. + pr_only = project.settings.get('pull_request_access_only', False) + + for repos in ['repos', 'docs/', 'tickets/', 'requests/']: + if repos == 'repos': + # Do not grant access to project enforcing the PR model + if pr_only or (global_pr_only and not project.is_fork): + continue + repos = '' + + config.append('repo %s%s' % (repos, project.fullname)) + if repos not in ['tickets/', 'requests/']: + config.append(' R = @all') + if project.committer_groups: + config.append(' RW+ = @%s' % ' @'.join( + [ + group.group_name + for group in project.committer_groups + ] + )) + config.append(' RW+ = %s' % project.user.user) + for user in project.committers: + # This should never be the case (that the project.user + # is in the committers) but better safe than sorry + if user.user != project.user.user: + config.append(' RW+ = %s' % user.user) + for deploykey in project.deploykeys: + access = 'R' + if deploykey.pushaccess: + access = 'RW+' + # Note: the replace of / with _ is because gitolite + # users can't contain a /. At first, this might look + # like deploy keys in a project called + # $namespace_$project would give access to the repos of + # a project $namespace/$project or vica versa, however + # this is NOT the case because we add the deploykey.id + # to the end of the deploykey name, which means it is + # unique. The project name is solely there to make it + # easier to determine what project created the deploykey + # for admins. + config.append(' %s = deploykey_%s_%s' % + (access, + werkzeug.secure_filename(project.fullname), + deploykey.id)) + config.append('') + + with open(configfile, 'w') as stream: + if preconfig: + stream.write(preconfig + '\n') + + for key, users in groups.iteritems(): + stream.write('@%s = %s\n' % (key, ' '.join(users))) + stream.write('\n') + + for row in config: + stream.write(row + '\n') + + if postconfig: + stream.write(postconfig + '\n') + + @staticmethod + def _get_gitolite_command(): + """ Return the gitolite command to run based on the info in the + configuration file. + """ + _log.info('Compiling the gitolite configuration') + gitolite_folder = pagure.APP.config.get('GITOLITE_HOME', None) + if gitolite_folder: + cmd = 'GL_RC=%s GL_BINDIR=%s gl-compile-conf' % ( + pagure.APP.config.get('GL_RC'), + pagure.APP.config.get('GL_BINDIR') + ) + _log.debug('Command: %s', cmd) + return cmd + + @classmethod + def generate_acls(cls): + """ Generate the gitolite configuration file for all repos + """ + _log.info('Refresh gitolite configuration') + cls.write_gitolite_acls( + pagure.SESSION, + pagure.APP.config['GITOLITE_CONFIG'], + preconf=pagure.APP.config.get('GITOLITE_PRE_CONFIG') or None, + postconf=pagure.APP.config.get('GITOLITE_POST_CONFIG') or None + ) + + cmd = cls._get_gitolite_command() + if cmd: + proc = subprocess.Popen( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=pagure.APP.config['GITOLITE_HOME'] + ) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + error_msg = ( + 'The command "{0}" failed with' + '\n\n out: "{1}\n\n err:"{2}"' + .format(' '.join(cmd), stdout, stderr)) + raise pagure.exceptions.PagureException(error_msg) + + +class Gitolite3Auth(Gitolite2Auth): + """ A gitolite 3 authentication module. """ + + @staticmethod + def _get_gitolite_command(): + """ Return the gitolite command to run based on the info in the + configuration file. + """ + _log.info('Compiling the gitolite configuration') + gitolite_folder = pagure.APP.config.get('GITOLITE_HOME', None) + if gitolite_folder: + cmd = 'HOME=%s gitolite compile && HOME=%s gitolite trigger '\ + 'POST_COMPILE' % (gitolite_folder, gitolite_folder) + _log.debug('Command: %s', cmd) + return cmd + + +class GitAuthTestHelper(GitAuthHelper): + """ Simple test auth module to check the auth customization system. """ + + @classmethod + def generate_acls(cls): + """ Print a statement when called, useful for debugging, only. """ + out = 'Called GitAuthTestHelper.generate_acls()' + print(out) + return out diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 6d838b8..ca04a0e 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -27,6 +27,7 @@ import pagure from pagure import APP import pagure.lib import pagure.lib.git +import pagure.lib.git_auth logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) _log = logging.getLogger(__name__) @@ -60,7 +61,9 @@ def gc_clean(): @conn.task def generate_gitolite_acls(): - pagure.lib.git._generate_gitolite_acls() + helper = pagure.lib.git_auth.get_git_auth_helper( + APP.config['GITOLITE_BACKEND']) + helper.generate_acls() gc_clean() diff --git a/setup.py b/setup.py index bcb139d..64f8804 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,14 @@ setup( entry_points=""" [pygments.styles] diffstyle = pagure.ui.diff_style:DiffStyle + [console_scripts] pagure-admin=pagure.cli.admin:main + + [pagure.git_auth.helpers] + test_auth = pagure.lib.git_auth:GitAuthTestHelper + gitolite2 = pagure.lib.git_auth:Gitolite2Auth + gitolite3 = pagure.lib.git_auth:Gitolite3Auth """, classifiers=[ 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', diff --git a/tests/test_pagure_flask_api_issue_comment.py b/tests/test_pagure_flask_api_issue_comment.py index cc923b6..afd07ae 100644 --- a/tests/test_pagure_flask_api_issue_comment.py +++ b/tests/test_pagure_flask_api_issue_comment.py @@ -43,7 +43,6 @@ class PagureFlaskApiIssueCommenttests(tests.Modeltests): pagure.APP.config['TICKETS_FOLDER'] = None - tests.create_projects(self.session) tests.create_projects_git(os.path.join(self.path, 'tickets')) tests.create_tokens(self.session) diff --git a/tests/test_pagure_lib_git.py b/tests/test_pagure_lib_git.py index 3764e62..84a6f53 100644 --- a/tests/test_pagure_lib_git.py +++ b/tests/test_pagure_lib_git.py @@ -21,7 +21,7 @@ import time import unittest import pygit2 -from mock import patch +from mock import patch, MagicMock sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) @@ -72,7 +72,8 @@ class PagureLibGittests(tests.Modeltests): outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -157,7 +158,8 @@ repo requests/forks/pingou/test3 with open(preconf, 'w') as stream: stream.write('# this is a header that is manually added') - pagure.lib.git.write_gitolite_acls( + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls( self.session, outputconf, preconf=preconf @@ -233,7 +235,8 @@ repo requests/somenamespace/test3 with open(postconf, 'w') as stream: stream.write('# end of generated configuration') - pagure.lib.git.write_gitolite_acls( + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls( self.session, outputconf, preconf=preconf, @@ -306,7 +309,8 @@ repo requests/somenamespace/test3 with open(postconf, 'w') as stream: stream.write('# end of generated configuration') - pagure.lib.git.write_gitolite_acls( + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls( self.session, outputconf, postconf=postconf @@ -404,7 +408,8 @@ repo requests/somenamespace/test3 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -514,7 +519,8 @@ repo requests/forks/pingou/test3 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -616,7 +622,8 @@ repo requests/forks/pingou/test3 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -767,7 +774,8 @@ repo requests/forks/pingou/test3 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -926,7 +934,8 @@ repo requests/forks/pingou/test2 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -1079,7 +1088,8 @@ repo requests/forks/pingou/test2 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -1198,7 +1208,8 @@ repo requests/forks/pingou/test2 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -1302,7 +1313,8 @@ repo requests/forks/pingou/test3 outputconf = os.path.join(self.path, 'test_gitolite.conf') - pagure.lib.git.write_gitolite_acls(self.session, outputconf) + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls(self.session, outputconf) self.assertTrue(os.path.exists(outputconf)) @@ -2974,7 +2986,12 @@ index 0000000..60f7480 pagure.lib.git.SESSION = self.session pagure.APP.config['GITOLITE_HOME'] = '/tmp' - pagure.lib.git._generate_gitolite_acls() + proc = MagicMock() + proc.communicate.return_value = (1, 2) + proc.returncode = 0 + popen.return_value = proc + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.generate_acls() popen.assert_called_with( 'HOME=/tmp gitolite compile && ' 'HOME=/tmp gitolite trigger POST_COMPILE',