Blame pagure/lib/tasks_mirror.py

Pierre-Yves Chibon 893d4f
# -*- coding: utf-8 -*-
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
"""
Pierre-Yves Chibon 893d4f
 (c) 2018 - Copyright Red Hat Inc
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
 Authors:
Pierre-Yves Chibon 893d4f
   Pierre-Yves Chibon <pingou@pingoured.fr></pingou@pingoured.fr>
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
"""
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 67d1cc
from __future__ import unicode_literals, absolute_import
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
import base64
Pierre-Yves Chibon 893d4f
import logging
Pierre-Yves Chibon 893d4f
import os
Pierre-Yves Chibon 893d4f
import stat
Pierre-Yves Chibon 893d4f
import struct
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
import six
Pierre-Yves Chibon 893d4f
import werkzeug
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
from celery import Celery
Pierre-Yves Chibon 893d4f
from cryptography import utils
Pierre-Yves Chibon 893d4f
from cryptography.hazmat.backends import default_backend
Pierre-Yves Chibon 893d4f
from cryptography.hazmat.primitives.asymmetric import rsa
Pierre-Yves Chibon 893d4f
from cryptography.hazmat.primitives import serialization
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 930073
import pagure.lib.query
Pierre-Yves Chibon 893d4f
from pagure.config import config as pagure_config
Pierre-Yves Chibon 1aac43
from pagure.lib.tasks_utils import pagure_task
Pierre-Yves Chibon 893d4f
from pagure.utils import ssh_urlpattern
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
# logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1})
Pierre-Yves Chibon 893d4f
_log = logging.getLogger(__name__)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
if os.environ.get("PAGURE_BROKER_URL"):  # pragma: no-cover
Pierre-Yves Chibon 9c2953
    broker_url = os.environ["PAGURE_BROKER_URL"]
Pierre-Yves Chibon 9c2953
elif pagure_config.get("BROKER_URL"):
Pierre-Yves Chibon 9c2953
    broker_url = pagure_config["BROKER_URL"]
Pierre-Yves Chibon 893d4f
else:
Pierre-Yves Chibon 9c2953
    broker_url = "redis://%s" % pagure_config["REDIS_HOST"]
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
conn = Celery("tasks_mirror", broker=broker_url, backend=broker_url)
Pierre-Yves Chibon 9c2953
conn.conf.update(pagure_config["CELERY_CONFIG"])
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
# Code from:
Pierre-Yves Chibon 893d4f
# https://github.com/pyca/cryptography/blob/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/primitives/serialization.py#L161
Pierre-Yves Chibon 893d4f
# Taken from upstream cryptography since the version we have is too old
Pierre-Yves Chibon 893d4f
# and doesn't have this code (yet)
Pierre-Yves Chibon 893d4f
def _ssh_write_string(data):
Pierre-Yves Chibon 893d4f
    return struct.pack(">I", len(data)) + data
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
def _ssh_write_mpint(value):
Pierre-Yves Chibon 893d4f
    data = utils.int_to_bytes(value)
Pierre-Yves Chibon 893d4f
    if six.indexbytes(data, 0) & 0x80:
Pierre-Yves Chibon 893d4f
        data = b"\x00" + data
Pierre-Yves Chibon 893d4f
    return _ssh_write_string(data)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
# Code from _openssh_public_key_bytes at:
Pierre-Yves Chibon 893d4f
# https://github.com/pyca/cryptography/tree/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/backends/openssl#L1616
Pierre-Yves Chibon 893d4f
# Taken from upstream cryptography since the version we have is too old
Pierre-Yves Chibon 893d4f
# and doesn't have this code (yet)
Pierre-Yves Chibon 893d4f
def _serialize_public_ssh_key(key):
Pierre-Yves Chibon 893d4f
    if isinstance(key, rsa.RSAPublicKey):
Pierre-Yves Chibon 893d4f
        public_numbers = key.public_numbers()
Pierre-Yves Chibon 893d4f
        return b"ssh-rsa " + base64.b64encode(
Pierre-Yves Chibon 9c2953
            _ssh_write_string(b"ssh-rsa")
Pierre-Yves Chibon 9c2953
            + _ssh_write_mpint(public_numbers.e)
Pierre-Yves Chibon 9c2953
            + _ssh_write_mpint(public_numbers.n)
Pierre-Yves Chibon 893d4f
        )
Pierre-Yves Chibon 893d4f
    else:
Pierre-Yves Chibon 893d4f
        # Since we only write RSA keys, drop the other serializations
Pierre-Yves Chibon 893d4f
        return
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
def _create_ssh_key(keyfile):
Pierre-Yves Chibon 9c2953
    """ Create the public and private ssh keys.
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    The specified file name will be the private key and the public one will
Pierre-Yves Chibon 893d4f
    be in a similar file name ending with a '.pub'.
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    """
Pierre-Yves Chibon 893d4f
    private_key = rsa.generate_private_key(
Pierre-Yves Chibon 9c2953
        public_exponent=65537, key_size=4096, backend=default_backend()
Pierre-Yves Chibon 893d4f
    )
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    private_pem = private_key.private_bytes(
Pierre-Yves Chibon 893d4f
        encoding=serialization.Encoding.PEM,
Pierre-Yves Chibon 893d4f
        format=serialization.PrivateFormat.TraditionalOpenSSL,
Pierre-Yves Chibon 9c2953
        encryption_algorithm=serialization.NoEncryption(),
Pierre-Yves Chibon 893d4f
    )
Pierre-Yves Chibon 9c2953
    with os.fdopen(
Pierre-Yves Chibon 9c2953
        os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb"
Pierre-Yves Chibon 9c2953
    ) as stream:
Pierre-Yves Chibon 893d4f
        stream.write(private_pem)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    public_key = private_key.public_key()
Pierre-Yves Chibon 893d4f
    public_pem = _serialize_public_ssh_key(public_key)
Pierre-Yves Chibon 893d4f
    if public_pem:
Pierre-Yves Chibon 9c2953
        with open(keyfile + ".pub", "wb") as stream:
Pierre-Yves Chibon 893d4f
            stream.write(public_pem)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
Pierre-Yves Chibon 893d4f
@pagure_task
Pierre-Yves Chibon 893d4f
def setup_mirroring(self, session, username, namespace, name):
Pierre-Yves Chibon 9c2953
    """ Setup the specified project for mirroring.
Pierre-Yves Chibon 9c2953
    """
Pierre-Yves Chibon 9c2953
    plugin = pagure.lib.plugins.get_plugin("Mirroring")
Pierre-Yves Chibon 893d4f
    plugin.db_object()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 930073
    project = pagure.lib.query._get_project(
Pierre-Yves Chibon 9c2953
        session, namespace=namespace, name=name, user=username
Pierre-Yves Chibon 9c2953
    )
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    public_key_name = werkzeug.secure_filename(project.fullname)
Pierre-Yves Chibon 9c2953
    ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    if not os.path.exists(ssh_folder):
Pierre-Yves Chibon 893d4f
        os.makedirs(ssh_folder, mode=0o700)
Pierre-Yves Chibon 893d4f
    else:
Pierre-Yves Chibon 893d4f
        if os.path.islink(ssh_folder):
Pierre-Yves Chibon 9c2953
            raise pagure.exceptions.PagureException("SSH folder is a link")
Pierre-Yves Chibon 893d4f
        folder_stat = os.stat(ssh_folder)
Pierre-Yves Chibon 893d4f
        filemode = stat.S_IMODE(folder_stat.st_mode)
Pierre-Yves Chibon 9c2953
        if filemode != int("0700", 8):
Pierre-Yves Chibon 893d4f
            raise pagure.exceptions.PagureException(
Pierre-Yves Chibon 9c2953
                "SSH folder had invalid permissions"
Pierre-Yves Chibon 9c2953
            )
Pierre-Yves Chibon 9c2953
        if (
Pierre-Yves Chibon 9c2953
            folder_stat.st_uid != os.getuid()
Pierre-Yves Chibon 9c2953
            or folder_stat.st_gid != os.getgid()
Pierre-Yves Chibon 9c2953
        ):
Pierre-Yves Chibon 893d4f
            raise pagure.exceptions.PagureException(
Pierre-Yves Chibon 9c2953
                "SSH folder does not belong to the user or group running "
Pierre-Yves Chibon 9c2953
                "this task"
Pierre-Yves Chibon 9c2953
            )
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    public_key_file = os.path.join(ssh_folder, "%s.pub" % public_key_name)
Pierre-Yves Chibon 9c2953
    _log.info("Public key of interest: %s", public_key_file)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    if os.path.exists(public_key_file):
Pierre-Yves Chibon 9c2953
        raise pagure.exceptions.PagureException("SSH key already exists")
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    _log.info("Creating public key")
Pierre-Yves Chibon 893d4f
    _create_ssh_key(os.path.join(ssh_folder, public_key_name))
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    with open(public_key_file) as stream:
Pierre-Yves Chibon 893d4f
        public_key = stream.read()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    if project.mirror_hook.public_key != public_key:
Pierre-Yves Chibon 9c2953
        _log.info("Updating information in the DB")
Pierre-Yves Chibon 893d4f
        project.mirror_hook.public_key = public_key
Pierre-Yves Chibon 893d4f
        session.add(project.mirror_hook)
Pierre-Yves Chibon 893d4f
        session.commit()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
Pierre-Yves Chibon 893d4f
@pagure_task
Pierre-Yves Chibon 893d4f
def teardown_mirroring(self, session, username, namespace, name):
Pierre-Yves Chibon 9c2953
    """ Stop the mirroring of the specified project.
Pierre-Yves Chibon 9c2953
    """
Pierre-Yves Chibon 9c2953
    plugin = pagure.lib.plugins.get_plugin("Mirroring")
Pierre-Yves Chibon 893d4f
    plugin.db_object()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 930073
    project = pagure.lib.query._get_project(
Pierre-Yves Chibon 9c2953
        session, namespace=namespace, name=name, user=username
Pierre-Yves Chibon 9c2953
    )
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    public_key_name = werkzeug.secure_filename(project.fullname)
Pierre-Yves Chibon 893d4f
    private_key_file = os.path.join(ssh_folder, public_key_name)
Pierre-Yves Chibon 9c2953
    public_key_file = os.path.join(ssh_folder, "%s.pub" % public_key_name)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    if os.path.exists(private_key_file):
Pierre-Yves Chibon 893d4f
        os.unlink(private_key_file)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    if os.path.exists(public_key_file):
Pierre-Yves Chibon 893d4f
        os.unlink(public_key_file)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    project.mirror_hook.public_key = None
Pierre-Yves Chibon 893d4f
    session.add(project.mirror_hook)
Pierre-Yves Chibon 893d4f
    session.commit()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
Pierre-Yves Chibon 893d4f
@pagure_task
Pierre-Yves Chibon 893d4f
def mirror_project(self, session, username, namespace, name):
Pierre-Yves Chibon 9c2953
    """ Does the actual mirroring of the specified project.
Pierre-Yves Chibon 9c2953
    """
Pierre-Yves Chibon 9c2953
    plugin = pagure.lib.plugins.get_plugin("Mirroring")
Pierre-Yves Chibon 893d4f
    plugin.db_object()
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 930073
    project = pagure.lib.query._get_project(
Pierre-Yves Chibon 9c2953
        session, namespace=namespace, name=name, user=username
Pierre-Yves Chibon 9c2953
    )
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    repofolder = pagure_config["GIT_FOLDER"]
Pierre-Yves Chibon 893d4f
    repopath = os.path.join(repofolder, project.path)
Pierre-Yves Chibon 893d4f
    if not os.path.exists(repopath):
Pierre-Yves Chibon d6adba
        _log.warning("Git folder not found at: %s, bailing", repopath)
Pierre-Yves Chibon 893d4f
        return
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 9c2953
    ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
Pierre-Yves Chibon 893d4f
    public_key_name = werkzeug.secure_filename(project.fullname)
Pierre-Yves Chibon 893d4f
    private_key_file = os.path.join(ssh_folder, public_key_name)
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 996c7e
    if not os.path.exists(private_key_file):
Pierre-Yves Chibon d6adba
        _log.warning("No %s key found, bailing", private_key_file)
Pierre-Yves Chibon 996c7e
        project.mirror_hook.last_log = "Private key not found on disk, bailing"
Pierre-Yves Chibon 996c7e
        session.add(project.mirror_hook)
Pierre-Yves Chibon 996c7e
        session.commit()
Pierre-Yves Chibon 996c7e
        return
Pierre-Yves Chibon 996c7e
Pierre-Yves Chibon 605eaa
    # Add the utility script allowing this feature to work on old(er) git.
Pierre-Yves Chibon 605eaa
    here = os.path.join(os.path.dirname(os.path.abspath(__file__)))
Pierre-Yves Chibon 605eaa
    script_file = os.path.join(here, "ssh_script.sh")
Pierre-Yves Chibon 605eaa
Pierre-Yves Chibon 893d4f
    # Get the list of remotes
Pierre-Yves Chibon 893d4f
    remotes = [
Pierre-Yves Chibon 893d4f
        remote.strip()
Pierre-Yves Chibon 9c2953
        for remote in project.mirror_hook.target.split("\n")
Pierre-Yves Chibon 9c2953
        if project.mirror_hook
Pierre-Yves Chibon 9c2953
        and remote.strip()
Pierre-Yves Chibon 893d4f
        and ssh_urlpattern.match(remote.strip())
Pierre-Yves Chibon 893d4f
    ]
Pierre-Yves Chibon 893d4f
Pierre-Yves Chibon 893d4f
    # Push
Pierre-Yves Chibon 893d4f
    logs = []
Pierre-Yves Chibon 605eaa
    for remote in remotes:
Pierre-Yves Chibon 893d4f
        _log.info(
Pierre-Yves Chibon 605eaa
            "Pushing to remote %s using key: %s", remote, private_key_file
Pierre-Yves Chibon 9c2953
        )
Pierre-Yves Chibon 893d4f
        (stdout, stderr) = pagure.lib.git.read_git_lines(
Pierre-Yves Chibon 605eaa
            ["push", "--mirror", remote],
Pierre-Yves Chibon 605eaa
            abspath=repopath,
Pierre-Yves Chibon 9c2953
            error=True,
Pierre-Yves Chibon 605eaa
            env={"SSHKEY": private_key_file, "GIT_SSH": script_file},
Pierre-Yves Chibon 9c2953
        )
Pierre-Yves Chibon 893d4f
        log = "Output from the push:\n  stdout: %s\n  stderr: %s" % (
Pierre-Yves Chibon 9c2953
            stdout,
Pierre-Yves Chibon 9c2953
            stderr,
Pierre-Yves Chibon 9c2953
        )
Pierre-Yves Chibon 893d4f
        logs.append(log)
Pierre-Yves Chibon 605eaa
Pierre-Yves Chibon 893d4f
    if logs:
Pierre-Yves Chibon 9c2953
        project.mirror_hook.last_log = "\n".join(logs)
Pierre-Yves Chibon 893d4f
        session.add(project.mirror_hook)
Pierre-Yves Chibon 893d4f
        session.commit()
Pierre-Yves Chibon 9c2953
        _log.info("\n".join(logs))