diff --git a/alembic/versions/5993f9240bcf_allow_mirroring_project_in.py b/alembic/versions/5993f9240bcf_allow_mirroring_project_in.py new file mode 100644 index 0000000..6265566 --- /dev/null +++ b/alembic/versions/5993f9240bcf_allow_mirroring_project_in.py @@ -0,0 +1,35 @@ +"""Allow mirroring project in + +Revision ID: 5993f9240bcf +Revises: 1f24c9c8efa5 +Create Date: 2018-12-14 10:00:05.281979 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5993f9240bcf' +down_revision = '1f24c9c8efa5' + + +def upgrade(): + ''' Add the column mirrored_from to the table projects. + ''' + op.add_column( + 'projects', + sa.Column('mirrored_from', sa.Text, nullable=True) + ) + op.add_column( + 'projects', + sa.Column('mirrored_from_last_log', sa.Text, nullable=True) + ) + + +def downgrade(): + ''' Remove the column mirrored_from from the table projects. + ''' + op.drop_column('projects', 'mirrored_from') + op.drop_column('projects', 'mirrored_from_last_log') diff --git a/files/mirror_project_in.py b/files/mirror_project_in.py new file mode 100644 index 0000000..5ee3b00 --- /dev/null +++ b/files/mirror_project_in.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +from __future__ import print_function +import os +import argparse +from datetime import datetime, timedelta + +from sqlalchemy.exc import SQLAlchemyError + +import pagure.config +import pagure.lib.query +import pagure.lib.notify +import pagure.lib.model as model + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print('Using configuration file `/etc/pagure/pagure.cfg`') + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + +_config = pagure.config.reload_config() + + +def main(check=False, debug=False): + ''' The function pulls in all the changes from upstream''' + + session = pagure.lib.query.create_session(_config['DB_URL']) + projects = session.query( + model.Project + ).filter( + model.Project.mirrored_from != None + ).all() + + for project in projects: + if debug: + print("Mirrorring %s" % project.fullname) + pagure.lib.git.mirror_pull_project(session, project, debug=debug) + + session.remove() + if debug: + print('Done') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Script to send email before the api token expires') + parser.add_argument( + '--check', dest='check', action='store_true', default=False, + help='Print the some output but does not send any email') + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the debugging output') + args = parser.parse_args() + main(debug=args.debug) diff --git a/pagure/forms.py b/pagure/forms.py index 82bb9c7..bd1efbe 100644 --- a/pagure/forms.py +++ b/pagure/forms.py @@ -156,6 +156,13 @@ class ProjectForm(ProjectFormSimplified): """ Form to create or edit project. """ name = wtforms.StringField('Project name *') + mirrored_from = wtforms.StringField( + "Mirror from URL", + [ + wtforms.validators.optional(), + wtforms.validators.Regexp(urlpattern, flags=re.IGNORECASE), + ], + ) create_readme = wtforms.BooleanField( "Create README", [wtforms.validators.optional()], diff --git a/pagure/lib/git.py b/pagure/lib/git.py index ba0de43..467c395 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -1211,10 +1211,12 @@ def read_output(cmd, abspath, input=None, keepends=False, error=False, **kw): cwd=abspath, **kw ) - (out, err) = procs.communicate(input) - out = out.decode("utf-8") - err = err.decode("utf-8") retcode = procs.wait() + (out, err) = procs.communicate(input) + if not isinstance(out, str): + out = out.decode("utf-8") + if not isinstance(err, str): + err = err.decode("utf-8") if retcode: print("ERROR: %s =-- %s" % (cmd, retcode)) print(out) @@ -2760,3 +2762,131 @@ def generate_archive(project, commit, tag, name, archive_fmt): raise pagure.exceptions.PagureException( "Un-support archive format requested: %s", archive_fmt ) + + +def mirror_pull_project(session, project, debug=False): + """ Mirror locally a project from a remote URL. """ + remote = project.mirrored_from + repopath = tempfile.mkdtemp(prefix="pagure-mirror_in-") + lclrepopath = pagure.utils.get_repo_path(project) + + def _run_command(command, logs): + _log.info("Running the command: %s" % command) + if debug: + print("Running the command: %s" % command) + print(" Running in: %s" % repopath) + (stdout, stderr) = pagure.lib.git.read_git_lines( + command, abspath=repopath, error=True + ) + log = "Output from %s:\n stdout: %s\n stderr: %s" % ( + command, + stdout, + stderr, + ) + logs.append(log) + if debug: + print(log) + return logs + + try: + # Pull + logs = [] + logs = _run_command(["clone", "--mirror", remote, "."], logs) + logs = _run_command(["remote", "add", "local", lclrepopath], logs) + + # Push the changes + _log.info("Pushing") + if debug: + print("Pushing to the local git repo") + extra = {} + if project.is_on_repospanner: + regioninfo = pagure_config["REPOSPANNER_REGIONS"][ + project.repospanner_region + ] + + extra.update( + { + "username": "pagure", + "repotype": "main", + "project_name": project.name, + "project_user": project.user.username + if project.is_fork + else "", + "project_namespace": project.namespace or "", + } + ) + args = [] + for opt in extra: + args.extend(["--extra", opt, extra[opt]]) + command = [ + "git", + "-c", + "protocol.ext.allow=always", + "push", + "ext::%s %s %s" + % ( + pagure_config["REPOBRIDGE_BINARY"], + " ".join(args), + project._repospanner_repo_name("main"), + ), + "--repo", + repopath, + ] + environ = { + "USER": "pagure", + "REPOBRIDGE_CONFIG": ":environment:", + "REPOBRIDGE_BASEURL": regioninfo["url"], + "REPOBRIDGE_CA": regioninfo["ca"], + "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"], + "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"], + } + else: + command = ["git", "push", "local", "--mirror"] + environ = {} + + _log.debug("Running a git push to %s", project.fullname) + env = os.environ.copy() + env["GL_USER"] = "pagure" + env["GL_BYPASS_ACCESS_CHECKS"] = "1" + if pagure_config.get("GITOLITE_HOME"): + env["HOME"] = pagure_config["GITOLITE_HOME"] + env.update(environ) + env.update(extra) + out = subprocess.check_output( + command, cwd=repopath, stderr=subprocess.STDOUT, env=env + ) + log = "Output from %s:" % command + logs.append(log) + logs.append(out) + _log.debug("Output: %s" % out) + + project.mirrored_from_last_log = "\n".join(logs) + session.add(project) + session.commit() + _log.info("\n".join(logs)) + except subprocess.CalledProcessError as err: + _log.debug( + "Rebase FAILED: {cmd} returned code {code} with the " + "following output: {output}".format( + cmd=err.cmd, code=err.returncode, output=err.output + ) + ) + # This should never really happen, since we control the repos, but + # this way, we can be sure to get the output logged + remotes = [] + for line in err.output.decode("utf-8").split("\n"): + _log.info("Remote line: %s", line) + if line.startswith("remote: "): + _log.debug("Remote: %s" % line) + remotes.append(line[len("remote: ") :].strip()) + if remotes: + _log.info("Remote rejected with: %s" % remotes) + raise pagure.exceptions.PagurePushDenied( + "Remote hook declined the push: %s" % "\n".join(remotes) + ) + else: + # Something else happened, pass the original + _log.exception("Error pushing. Output: %s", err.output) + raise + finally: + shutil.rmtree(repopath) diff --git a/pagure/lib/model.py b/pagure/lib/model.py index d0562a8..85e5695 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -380,6 +380,8 @@ class Project(BASE): _reports = sa.Column(sa.Text, nullable=True) _notifications = sa.Column(sa.Text, nullable=True) _close_status = sa.Column(sa.Text, nullable=True) + mirrored_from = sa.Column(sa.Text, nullable=True) + mirrored_from_last_log = sa.Column(sa.Text, nullable=True) date_created = sa.Column( sa.DateTime, nullable=False, default=datetime.datetime.utcnow diff --git a/pagure/lib/query.py b/pagure/lib/query.py index 8a5c2b2..790989f 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -1631,6 +1631,7 @@ def new_project( avatar_email=None, parent_id=None, add_readme=False, + mirrored_from=None, userobj=None, prevent_40_chars=False, namespace=None, @@ -1712,6 +1713,7 @@ def new_project( avatar_email=avatar_email if avatar_email else None, user_id=user_obj.id, parent_id=parent_id, + mirrored_from=mirrored_from, private=private, hook_token=pagure.lib.login.id_generator(40), ) diff --git a/pagure/templates/new_project.html b/pagure/templates/new_project.html index 8a4b930..ed638b7 100644 --- a/pagure/templates/new_project.html +++ b/pagure/templates/new_project.html @@ -23,6 +23,7 @@ {% if config.get('PRIVATE_PROJECTS', False) %} {{ render_bootstrap_field(form.private, field_description="To mark the repo private") }} {% endif %} + {{ render_bootstrap_field(form.mirrored_from, field_description="Mirror this project from another git server") }} {{ render_bootstrap_field(form.create_readme, field_description="Create a README file automatically") }} {% if form.repospanner_region %} {{ render_bootstrap_field(form.repospanner_region, field_description="repoSpanner region to create the project in") }} @@ -36,6 +37,15 @@ +
+
+
+

+ Note that Pagure only supports mirroring from a public server. +

+
+
+
{% endblock %} @@ -56,6 +66,20 @@ $('#private').change(function(){ $('#namespace').removeAttr("disabled"); } }); +function update_if_mirror() { + if ($('#mirrored_from').val()){ + $('#create_readme').attr("disabled", "disabled"); + $('#create_readme').prop('checked', false); + $('#mirrored_doc').show(); + } else { + $('#create_readme').removeAttr("disabled"); + $('#mirrored_doc').hide(); + } +}; +$('#mirrored_from').keyup(function(){ + update_if_mirror(); +}); +update_if_mirror(); {% endblock %} {% endif %} diff --git a/pagure/templates/repo_info.html b/pagure/templates/repo_info.html index e2f540b..fa3da23 100644 --- a/pagure/templates/repo_info.html +++ b/pagure/templates/repo_info.html @@ -279,7 +279,14 @@ {% endif %} - {% if g.repo_obj and g.repo_obj.is_empty %} + {% if g.repo_obj and g.repo_obj.is_empty and repo.mirrored_from %} + + {% elif g.repo_obj and g.repo_obj.is_empty %}