diff --git a/pagure/default_config.py b/pagure/default_config.py index 49231f0..5304200 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -105,6 +105,7 @@ WEBHOOK_CELERY_QUEUE = 'pagure_webhook' LOGCOM_CELERY_QUEUE = 'pagure_logcom' LOADJSON_CELERY_QUEUE = 'pagure_loadjson' CI_CELERY_QUEUE = 'pagure_ci' +MIRRORING_QUEUE = 'pagure_mirror' # Number of items displayed per page ITEM_PER_PAGE = 48 @@ -126,6 +127,9 @@ REDIS_PORT = 6379 REDIS_DB = 0 EVENTSOURCE_PORT = 8080 +# Folder where to place the ssh keys for the mirroring feature +MIRROR_SSHKEYS_FOLDER = '/var/lib/pagure/sshkeys/' + # Folder containing to the git repos # Note that this must be exactly the same as GL_REPO_BASE in gitolite.rc GIT_FOLDER = os.path.join( diff --git a/pagure/hooks/files/mirror.py b/pagure/hooks/files/mirror.py new file mode 100755 index 0000000..769a3a5 --- /dev/null +++ b/pagure/hooks/files/mirror.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python + + +"""Pagure specific hook to mirror a repo to another location. +""" +from __future__ import unicode_literals, print_function + + +import logging +import os +import sys + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure.config # noqa: E402 +import pagure.exceptions # noqa: E402 +import pagure.lib # noqa: E402 +import pagure.lib.tasks_mirror # noqa: E402 +import pagure.ui.plugins # noqa: E402 + + +_log = logging.getLogger(__name__) +_config = pagure.config.config +abspath = os.path.abspath(os.environ['GIT_DIR']) + + +def main(args): + + repo = pagure.lib.git.get_repo_name(abspath) + username = pagure.lib.git.get_username(abspath) + namespace = pagure.lib.git.get_repo_namespace(abspath) + if _config.get('HOOK_DEBUG', False): + print('repo:', repo) + print('user:', username) + print('namespace:', namespace) + + session = pagure.lib.create_session(_config['DB_URL']) + project = pagure.lib._get_project( + session, repo, user=username, namespace=namespace) + + if not project: + print('Could not find a project corresponding to this git repo') + session.close() + return 1 + + pagure.lib.tasks_mirror.mirror_project.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + session.close() + return 0 + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/pagure/hooks/mirror_hook.py b/pagure/hooks/mirror_hook.py new file mode 100644 index 0000000..b874738 --- /dev/null +++ b/pagure/hooks/mirror_hook.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2016-2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + + +import sqlalchemy as sa +import wtforms + +try: + from flask_wtf import FlaskForm +except ImportError: + from flask_wtf import Form as FlaskForm +from sqlalchemy.orm import relation +from sqlalchemy.orm import backref + +import pagure.lib.tasks_mirror +from pagure.hooks import BaseHook, RequiredIf +from pagure.lib.model import BASE, Project +from pagure.utils import get_repo_path, ssh_urlpattern + + +class MirrorTable(BASE): + """ Stores information about the mirroring hook deployed on a project. + + Table -- mirror_pagure + """ + + __tablename__ = 'hook_mirror' + + id = sa.Column(sa.Integer, primary_key=True) + project_id = sa.Column( + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + unique=True, + index=True) + + active = sa.Column(sa.Boolean, nullable=False, default=False) + + public_key = sa.Column(sa.Text, nullable=True) + target = sa.Column(sa.Text, nullable=True) + last_log = sa.Column(sa.Text, nullable=True) + + project = relation( + 'Project', remote_side=[Project.id], + backref=backref( + 'mirror_hook', cascade="delete, delete-orphan", + single_parent=True, uselist=False) + ) + + +class CustomRegexp(wtforms.validators.Regexp): + + def __init__(self, *args, **kwargs): + self.optional = kwargs.get('optional') or False + if self.optional: + kwargs.pop('optional') + super(CustomRegexp, self).__init__(*args, **kwargs) + + def __call__(self, form, field): + if self.optional: + if field.data: + return super(CustomRegexp, self).__call__(form, field) + else: + return super(CustomRegexp, self).__call__(form, field) + + +class MirrorForm(FlaskForm): + ''' Form to configure the mirror hook. ''' + active = wtforms.BooleanField( + 'Active', + [wtforms.validators.Optional()] + ) + + target = wtforms.TextField( + 'Git repo to mirror to', + [ + RequiredIf('active'), + CustomRegexp(ssh_urlpattern, optional=True), + ] + ) + + public_key = wtforms.TextAreaField( + 'Public SSH key', + [wtforms.validators.Optional()] + ) + last_log = wtforms.TextAreaField( + 'Log of the last sync:', + [wtforms.validators.Optional()] + ) + + +DESCRIPTION = ''' +Pagure specific hook to mirror a repo hosted on pagure to another location. + +The first field below should contain the URL to be set in the git configuration +as the URL of the git repository to mirror to. +It's format is going to be something like: + + @: + +The public SSH key is being generated by pagure and will be available in this +page shortly after the activation of this hook. Just refresh the page until +it shows up. + +Finally the log of the last sync at the bottom is meant. +''' + + +class MirrorHook(BaseHook): + ''' Mirror hook. ''' + + name = 'Mirroring' + description = DESCRIPTION + form = MirrorForm + db_object = MirrorTable + backref = 'mirror_hook' + form_fields = ['active', 'target', 'public_key', 'last_log'] + form_fields_readonly = ['public_key', 'last_log'] + + @classmethod + def install(cls, project, dbobj): + ''' Method called to install the hook for a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + pagure.lib.tasks_mirror.setup_mirroring.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + repopaths = [get_repo_path(project)] + cls.base_install(repopaths, dbobj, 'mirror', 'mirror.py') + + @classmethod + def remove(cls, project): + ''' Method called to remove the hook of a project. + + :arg project: a ``pagure.model.Project`` object to which the hook + should be installed + + ''' + pagure.lib.tasks_mirror.teardown_mirroring.delay( + username=project.user.user if project.is_fork else None, + namespace=project.namespace, + name=project.name) + + repopaths = [get_repo_path(project)] + cls.base_remove(repopaths, 'mirror') diff --git a/pagure/lib/tasks_mirror.py b/pagure/lib/tasks_mirror.py new file mode 100644 index 0000000..f64d423 --- /dev/null +++ b/pagure/lib/tasks_mirror.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +import base64 +import logging +import os +import shutil +import stat +import struct +import tempfile + +import pygit2 +import six +import werkzeug + +from celery import Celery +from cryptography import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +import pagure.lib +from pagure.config import config as pagure_config +from pagure.lib.tasks import pagure_task +from pagure.utils import ssh_urlpattern + +# logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1}) +_log = logging.getLogger(__name__) + + +if os.environ.get('PAGURE_BROKER_URL'): # pragma: no-cover + broker_url = os.environ['PAGURE_BROKER_URL'] +elif pagure_config.get('BROKER_URL'): + broker_url = pagure_config['BROKER_URL'] +else: + broker_url = 'redis://%s' % pagure_config['REDIS_HOST'] + +conn = Celery('tasks_mirror', broker=broker_url, backend=broker_url) +conn.conf.update(pagure_config['CELERY_CONFIG']) + + +# Code from: +# https://github.com/pyca/cryptography/blob/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/primitives/serialization.py#L161 +# Taken from upstream cryptography since the version we have is too old +# and doesn't have this code (yet) +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from _openssh_public_key_bytes at: +# https://github.com/pyca/cryptography/tree/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/backends/openssl#L1616 +# Taken from upstream cryptography since the version we have is too old +# and doesn't have this code (yet) +def _serialize_public_ssh_key(key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + else: + # Since we only write RSA keys, drop the other serializations + return + + +def _create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + with os.fdopen(os.open( + keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb')\ + as stream: + stream.write(private_pem) + + public_key = private_key.public_key() + public_pem = _serialize_public_ssh_key(public_key) + if public_pem: + with open(keyfile + '.pub', 'wb') as stream: + stream.write(public_pem) + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def setup_mirroring(self, session, username, namespace, name): + ''' Setup the specified project for mirroring. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder, mode=0o700) + else: + if os.path.islink(ssh_folder): + raise pagure.exceptions.PagureException( + 'SSH folder is a link') + folder_stat = os.stat(ssh_folder) + filemode = stat.S_IMODE(folder_stat.st_mode) + if filemode != int('0700', 8): + raise pagure.exceptions.PagureException( + 'SSH folder had invalid permissions') + if folder_stat.st_uid != os.getuid() \ + or folder_stat.st_gid != os.getgid(): + raise pagure.exceptions.PagureException( + 'SSH folder does not belong to the user or group running ' + 'this task') + + public_key_file = os.path.join(ssh_folder, '%s.pub' % public_key_name) + _log.info('Public key of interest: %s', public_key_file) + + if os.path.exists(public_key_file): + raise pagure.exceptions.PagureException('SSH key already exists') + + _log.info('Creating public key') + _create_ssh_key(os.path.join(ssh_folder, public_key_name)) + + with open(public_key_file) as stream: + public_key = stream.read() + + if project.mirror_hook.public_key != public_key: + _log.info('Updating information in the DB') + project.mirror_hook.public_key = public_key + session.add(project.mirror_hook) + session.commit() + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def teardown_mirroring(self, session, username, namespace, name): + ''' Stop the mirroring of the specified project. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exists(private_key_file): + os.unlink(private_key_file) + + if os.path.exists(public_key_file): + os.unlink(public_key_file) + + project.mirror_hook.public_key = None + session.add(project.mirror_hook) + session.commit() + + +@conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True) +@pagure_task +def mirror_project(self, session, username, namespace, name): + ''' Does the actual mirroring of the specified project. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + plugin.db_object() + + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=username) + + repofolder = pagure_config['GIT_FOLDER'] + repopath = os.path.join(repofolder, project.path) + if not os.path.exists(repopath): + _log.info('Git folder not found at: %s, bailing', repopath) + return + + newpath = tempfile.mkdtemp(prefix='pagure-mirror-') + pygit2.clone_repository(repopath, newpath) + + ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER'] + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in project.mirror_hook.target.split('\n') + if project.mirror_hook and remote.strip() + and ssh_urlpattern.match(remote.strip()) + ] + + # Add the remotes + for idx, remote in enumerate(remotes): + remote_name = '%s_%s' % (public_key_name, idx) + _log.info('Adding remote %s as %s', remote, remote_name) + (stdout, stderr) = pagure.lib.git.read_git_lines( + ['remote', 'add', remote_name, remote, '--mirror=push'], + abspath=newpath, error=True) + _log.info( + "Output from git remote add:\n stdout: %s\n stderr: %s", + stdout, stderr) + + # Push + logs = [] + for idx, remote in enumerate(remotes): + remote_name = '%s_%s' % (public_key_name, idx) + _log.info( + 'Pushing to remote %s using key: %s', remote_name, + private_key_file) + (stdout, stderr) = pagure.lib.git.read_git_lines( + ['push', remote_name], + abspath=newpath, error=True, + env={'GIT_SSH_COMMAND': 'ssh -i %s' % private_key_file}) + log = "Output from the push:\n stdout: %s\n stderr: %s" % ( + stdout, stderr) + logs.append(log) + if logs: + project.mirror_hook.last_log = '\n'.join(logs) + session.add(project.mirror_hook) + session.commit() + _log.info('\n'.join(logs)) + + # Remove the clone + shutil.rmtree(newpath) diff --git a/pagure/utils.py b/pagure/utils.py index 1d647d8..101f99a 100644 --- a/pagure/utils.py +++ b/pagure/utils.py @@ -344,6 +344,47 @@ urlregex = re.compile( urlpattern = re.compile(urlregex) +ssh_urlregex = re.compile( + u"^" + # protocol identifier + u"(?:(?:(git\+)?ssh)://)" + # user:pass authentication + u"(?:\S+(?::\S*)?@)?" + u"(?:" + u"(?P" + # IP address exclusion + # private & local networks + u"(?:(?:10|127)" + ip_middle_octet + u"{2}" + ip_last_octet + u")|" + u"(?:(?:169\.254|192\.168)" + ip_middle_octet + ip_last_octet + u")|" + u"(?:172\.(?:1[6-9]|2\d|3[0-1])" + ip_middle_octet + ip_last_octet + u"))" + u"|" + # IP address dotted notation octets + # excludes loopback network 0.0.0.0 + # excludes reserved space >= 224.0.0.0 + # excludes network & broadcast addresses + # (first & last IP address of each class) + u"(?P" + u"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + u"" + ip_middle_octet + u"{2}" + u"" + ip_last_octet + u")" + u"|" + # host name + u"(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" + # domain name + u"(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" + # TLD identifier + u"(?:\.(?:[a-z\u00a1-\uffff]{2,}))" + u")" + # port number + u"(?::\d{2,5})?" + # resource path + u"(?:/\S*)?" + u"$", + re.UNICODE | re.IGNORECASE +) +ssh_urlpattern = re.compile(ssh_urlregex) + + def get_repo_path(repo): """ Return the path of the git repository corresponding to the provided Repository object from the DB. diff --git a/tests/__init__.py b/tests/__init__.py index c325303..8c2e5ef 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -362,6 +362,7 @@ class SimplePagureTest(unittest.TestCase): pagure_config.update(reload_config()) imp.reload(pagure.lib.tasks) + imp.reload(pagure.lib.tasks_mirror) imp.reload(pagure.lib.tasks_services) self._app = pagure.flask_app.create_app({'DB_URL': self.dbpath}) diff --git a/tests/test_pagure_flask_ui_plugins.py b/tests/test_pagure_flask_ui_plugins.py index ab5f474..1c68bbf 100644 --- a/tests/test_pagure_flask_ui_plugins.py +++ b/tests/test_pagure_flask_ui_plugins.py @@ -50,6 +50,7 @@ class PagureFlaskPluginstests(tests.SimplePagureTest): 'Fedmsg', 'IRC', 'Mail', + 'Mirroring', 'Pagure', 'Pagure CI', 'Pagure requests', diff --git a/tests/test_pagure_flask_ui_plugins_mirror.py b/tests/test_pagure_flask_ui_plugins_mirror.py new file mode 100644 index 0000000..6ba8ef4 --- /dev/null +++ b/tests/test_pagure_flask_ui_plugins_mirror.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +__requires__ = ['SQLAlchemy >= 0.8'] + +import unittest +import sys +import os + +from mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure.lib +import tests + + +class PagureFlaskPluginMirrortests(tests.Modeltests): + """ Tests for mirror plugin of pagure """ + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskPluginMirrortests, self).setUp() + + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos')) + + def test_plugin_mirror_no_csrf(self): + """ Test setting up the mirror plugin with no csrf. """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + output = self.app.get('/test/settings/Mirroring') + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + data = {} + + output = self.app.post('/test/settings/Mirroring', data=data) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_plugin_mirror_no_data(self): + """ Test the setting up the mirror plugin when there are no data + provided in the request. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = {'csrf_token': csrf_token} + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', output_text) + self.assertIn( + ' Hook Mirroring deactivated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_plugin_mirror_invalid_target(self): + """ Test the setting up the mirror plugin when there are the target + provided is invalid. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = { + 'csrf_token': csrf_token, + 'active': True, + 'target': 'https://host.org/target', + } + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + if self.get_wtforms_version() >= (2, 2): + self.assertIn( + '' + '\nInvalid input.', + output_text) + else: + self.assertIn( + '\n' + 'Invalid input.', output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + def test_setting_up_mirror(self): + """ Test the setting up the mirror plugin. + """ + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + data = { + 'csrf_token': csrf_token, + 'active': True, + 'target': 'ssh://user@host.org/target', + } + + # With the git repo + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', + output_text) + self.assertIn( + ' Hook Mirroring activated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output_text) + + self.assertTrue(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + self.assertTrue(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive'))) + + def test_plugin_mirror_deactivate(self): + """ Test the deactivating the mirror plugin. + """ + self.test_setting_up_mirror() + + user = tests.FakeUser(username='pingou') + with tests.user_set(self.app.application, user): + csrf_token = self.get_csrf() + + # De-Activate hook + data = {'csrf_token': csrf_token} + output = self.app.post( + '/test/settings/Mirroring', data=data, follow_redirects=True) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings - test - Pagure', + output_text) + self.assertIn( + ' Hook Mirroring deactivated', + output_text) + + output = self.app.get('/test/settings/Mirroring', data=data) + self.assertEqual(output.status_code, 200) + output_text = output.get_data(as_text=True) + self.assertIn( + 'Settings Mirroring - test - Pagure', + output_text) + self.assertIn( + '', output.get_data(as_text=True)) + + self.assertFalse(os.path.exists(os.path.join( + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.mirror'))) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_pagure_flask_util.py b/tests/test_pagure_flask_util.py new file mode 100644 index 0000000..5eec180 --- /dev/null +++ b/tests/test_pagure_flask_util.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +__requires__ = ['SQLAlchemy >= 0.8'] +import pkg_resources + +import unittest +import sys +import os + +from mock import patch, MagicMock + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +from pagure.utils import ssh_urlpattern +import tests + + +class PagureUtilSSHPatterntests(tests.Modeltests): + """ Tests for the ssh_urlpattern in pagure.util """ + + def test_ssh_pattern_valid(self): + """ Test the ssh_urlpattern with valid patterns. """ + patterns = [ + 'ssh://user@host.com/repo.git', + 'git+ssh://user@host.com/repo.git', + 'ssh://host.com/repo.git' + 'git+ssh://host.com/repo.git', + 'ssh://127.0.0.1/repo.git', + 'git+ssh://127.0.0.1/repo.git', + ] + for pattern in patterns: + print(pattern) + self.assertTrue(ssh_urlpattern.match(pattern)) + + + def test_ssh_pattern_invalid(self): + """ Test the ssh_urlpattern with invalid patterns. """ + patterns = [ + 'http://user@host.com/repo.git', + 'git+http://user@host.com/repo.git', + 'https://user@host.com/repo.git', + 'git+https://user@host.com/repo.git', + 'ssh://localhost/repo.git', + 'git+ssh://localhost/repo.git', + 'ssh://0.0.0.0/repo.git', + 'git+ssh://0.0.0.0/repo.git', + ] + for pattern in patterns: + print(pattern) + self.assertFalse(ssh_urlpattern.match(pattern)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_pagure_lib_task_mirror.py b/tests/test_pagure_lib_task_mirror.py new file mode 100644 index 0000000..f4752df --- /dev/null +++ b/tests/test_pagure_lib_task_mirror.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +import datetime +import os +import shutil +import sys +import tempfile +import time +import unittest + +import pygit2 +import six +from mock import patch, MagicMock, call + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure +import pagure.lib.git +import tests + +import pagure.lib.tasks_mirror + + +class PagureLibTaskMirrortests(tests.Modeltests): + """ Tests for pagure.lib.task_mirror """ + + maxDiff = None + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureLibTaskMirrortests, self).setUp() + + pagure.config.config['REQUESTS_FOLDER'] = None + self.sshkeydir = os.path.join(self.path, 'sshkeys') + pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir + + tests.create_projects(self.session) + + def test_create_ssh_key(self): + """ Test the _create_ssh_key method. """ + # before + self.assertFalse(os.path.exists(self.sshkeydir)) + os.mkdir(self.sshkeydir) + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + + keyfile = os.path.join(self.sshkeydir, 'testkey') + pagure.lib.tasks_mirror._create_ssh_key(keyfile) + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'testkey', u'testkey.pub'] + ) + + def test_setup_mirroring(self): + """ Test the setup_mirroring method. """ + + # before + self.assertFalse(os.path.exists(self.sshkeydir)) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + pagure.lib.tasks_mirror.setup_mirroring( + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + def test_setup_mirroring_ssh_folder_exists_wrong_permissions(self): + """ Test the setup_mirroring method. """ + + os.makedirs(self.sshkeydir) + + # before + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + def test_setup_mirroring_ssh_folder_symlink(self): + """ Test the setup_mirroring method. """ + + os.symlink( + self.path, + self.sshkeydir + ) + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'attachments', u'config', u'forks', u'releases', + u'remotes', u'repos', u'sshkeys'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'attachments', u'config', u'forks', u'releases', + u'remotes', u'repos', u'sshkeys'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + @patch('os.getuid', MagicMock(return_value=450)) + def test_setup_mirroring_ssh_folder_owner(self): + """ Test the setup_mirroring method. """ + os.makedirs(self.sshkeydir, mode=0o700) + + # before + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook) + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + +class PagureLibTaskMirrorSetuptests(tests.Modeltests): + """ Tests for pagure.lib.task_mirror """ + + maxDiff = None + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureLibTaskMirrorSetuptests, self).setUp() + + pagure.config.config['REQUESTS_FOLDER'] = None + self.sshkeydir = os.path.join(self.path, 'sshkeys') + pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir + + tests.create_projects(self.session) + project = pagure.lib.get_authorized_project(self.session, 'test') + + # Install the plugin at the DB level + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj.target = 'ssh://user@localhost.localdomain/foobar.git' + dbobj.project_id = project.id + self.session.add(dbobj) + self.session.commit() + + pagure.lib.tasks_mirror.setup_mirroring( + username=None, + namespace=None, + name='test') + + def test_setup_mirroring_twice(self): + """ Test the setup_mirroring method. """ + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + before_key = project.mirror_hook.public_key + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + self.assertRaises( + pagure.exceptions.PagureException, + pagure.lib.tasks_mirror.setup_mirroring, + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertEqual(project.mirror_hook.public_key, before_key) + + def test_teardown_mirroring(self): + """ Test the teardown_mirroring method. """ + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + pagure.lib.tasks_mirror.teardown_mirroring( + username=None, + namespace=None, + name='test') + + # after + self.session = pagure.lib.create_session(self.dbpath) + self.assertEqual(sorted(os.listdir(self.sshkeydir)), []) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNone(project.mirror_hook.public_key) + + @patch( + 'tempfile.mkdtemp', + MagicMock(return_value='/tmp/pagure-mirror-fdgqcF')) + @patch('pagure.lib.git.read_git_lines') + def test_mirror_project(self,rgl): + """ Test the mirror_project method. """ + rgl.return_value = ('stdout', 'stderr') + tests.create_projects_git( + os.path.join(self.path, 'repos'), bare=True) + + # before + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + pagure.lib.tasks_mirror.mirror_project( + username=None, + namespace=None, + name='test') + + # after + self.assertEqual( + sorted(os.listdir(self.sshkeydir)), + [u'test', u'test.pub'] + ) + project = pagure.lib.get_authorized_project(self.session, 'test') + self.assertIsNotNone(project.mirror_hook.public_key) + self.assertTrue( + project.mirror_hook.public_key.startswith('ssh-rsa ')) + + calls = [ + call( + [ + u'remote', u'add', u'test_0', + u'ssh://user@localhost.localdomain/foobar.git', + u'--mirror=push' + ], + abspath=u'/tmp/pagure-mirror-fdgqcF', + error=True + ), + call( + [u'push', u'test_0'], + abspath=u'/tmp/pagure-mirror-fdgqcF', + env={ + u'GIT_SSH_COMMAND': u'ssh -i %s/sshkeys/test' % self.path + }, + error=True + ) + ] + + self.assertEqual(rgl.call_count, 2) + self.assertEqual( + calls, + rgl.mock_calls + ) + + +if __name__ == '__main__': + unittest.main(verbosity=2)