|
Pierre-Yves Chibon |
893d4f |
# -*- coding: utf-8 -*-
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
"""
|
|
Pierre-Yves Chibon |
893d4f |
(c) 2018 - Copyright Red Hat Inc
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
Authors:
|
|
Pierre-Yves Chibon |
893d4f |
Pierre-Yves Chibon <pingou@pingoured.fr></pingou@pingoured.fr>
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
"""
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
67d1cc |
from __future__ import unicode_literals, absolute_import
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
import base64
|
|
Pierre-Yves Chibon |
893d4f |
import logging
|
|
Pierre-Yves Chibon |
893d4f |
import os
|
|
Pierre-Yves Chibon |
893d4f |
import stat
|
|
Pierre-Yves Chibon |
893d4f |
import struct
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
import six
|
|
Pierre-Yves Chibon |
893d4f |
import werkzeug
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
from celery import Celery
|
|
Pierre-Yves Chibon |
893d4f |
from cryptography import utils
|
|
Pierre-Yves Chibon |
893d4f |
from cryptography.hazmat.backends import default_backend
|
|
Pierre-Yves Chibon |
893d4f |
from cryptography.hazmat.primitives.asymmetric import rsa
|
|
Pierre-Yves Chibon |
893d4f |
from cryptography.hazmat.primitives import serialization
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
930073 |
import pagure.lib.query
|
|
Pierre-Yves Chibon |
893d4f |
from pagure.config import config as pagure_config
|
|
Pierre-Yves Chibon |
1aac43 |
from pagure.lib.tasks_utils import pagure_task
|
|
Pierre-Yves Chibon |
893d4f |
from pagure.utils import ssh_urlpattern
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
# logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1})
|
|
Pierre-Yves Chibon |
893d4f |
_log = logging.getLogger(__name__)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
if os.environ.get("PAGURE_BROKER_URL"): # pragma: no-cover
|
|
Pierre-Yves Chibon |
9c2953 |
broker_url = os.environ["PAGURE_BROKER_URL"]
|
|
Pierre-Yves Chibon |
9c2953 |
elif pagure_config.get("BROKER_URL"):
|
|
Pierre-Yves Chibon |
9c2953 |
broker_url = pagure_config["BROKER_URL"]
|
|
Pierre-Yves Chibon |
893d4f |
else:
|
|
Pierre-Yves Chibon |
9c2953 |
broker_url = "redis://%s" % pagure_config["REDIS_HOST"]
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
conn = Celery("tasks_mirror", broker=broker_url, backend=broker_url)
|
|
Pierre-Yves Chibon |
9c2953 |
conn.conf.update(pagure_config["CELERY_CONFIG"])
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
# Code from:
|
|
Pierre-Yves Chibon |
893d4f |
# https://github.com/pyca/cryptography/blob/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/primitives/serialization.py#L161
|
|
Pierre-Yves Chibon |
893d4f |
# Taken from upstream cryptography since the version we have is too old
|
|
Pierre-Yves Chibon |
893d4f |
# and doesn't have this code (yet)
|
|
Pierre-Yves Chibon |
893d4f |
def _ssh_write_string(data):
|
|
Pierre-Yves Chibon |
893d4f |
return struct.pack(">I", len(data)) + data
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
def _ssh_write_mpint(value):
|
|
Pierre-Yves Chibon |
893d4f |
data = utils.int_to_bytes(value)
|
|
Pierre-Yves Chibon |
893d4f |
if six.indexbytes(data, 0) & 0x80:
|
|
Pierre-Yves Chibon |
893d4f |
data = b"\x00" + data
|
|
Pierre-Yves Chibon |
893d4f |
return _ssh_write_string(data)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
# Code from _openssh_public_key_bytes at:
|
|
Pierre-Yves Chibon |
893d4f |
# https://github.com/pyca/cryptography/tree/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/backends/openssl#L1616
|
|
Pierre-Yves Chibon |
893d4f |
# Taken from upstream cryptography since the version we have is too old
|
|
Pierre-Yves Chibon |
893d4f |
# and doesn't have this code (yet)
|
|
Pierre-Yves Chibon |
893d4f |
def _serialize_public_ssh_key(key):
|
|
Pierre-Yves Chibon |
893d4f |
if isinstance(key, rsa.RSAPublicKey):
|
|
Pierre-Yves Chibon |
893d4f |
public_numbers = key.public_numbers()
|
|
Pierre-Yves Chibon |
893d4f |
return b"ssh-rsa " + base64.b64encode(
|
|
Pierre-Yves Chibon |
9c2953 |
_ssh_write_string(b"ssh-rsa")
|
|
Pierre-Yves Chibon |
9c2953 |
+ _ssh_write_mpint(public_numbers.e)
|
|
Pierre-Yves Chibon |
9c2953 |
+ _ssh_write_mpint(public_numbers.n)
|
|
Pierre-Yves Chibon |
893d4f |
)
|
|
Pierre-Yves Chibon |
893d4f |
else:
|
|
Pierre-Yves Chibon |
893d4f |
# Since we only write RSA keys, drop the other serializations
|
|
Pierre-Yves Chibon |
893d4f |
return
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
def _create_ssh_key(keyfile):
|
|
Pierre-Yves Chibon |
9c2953 |
""" Create the public and private ssh keys.
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
The specified file name will be the private key and the public one will
|
|
Pierre-Yves Chibon |
893d4f |
be in a similar file name ending with a '.pub'.
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
"""
|
|
Pierre-Yves Chibon |
893d4f |
private_key = rsa.generate_private_key(
|
|
Pierre-Yves Chibon |
9c2953 |
public_exponent=65537, key_size=4096, backend=default_backend()
|
|
Pierre-Yves Chibon |
893d4f |
)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
private_pem = private_key.private_bytes(
|
|
Pierre-Yves Chibon |
893d4f |
encoding=serialization.Encoding.PEM,
|
|
Pierre-Yves Chibon |
893d4f |
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
|
Pierre-Yves Chibon |
9c2953 |
encryption_algorithm=serialization.NoEncryption(),
|
|
Pierre-Yves Chibon |
893d4f |
)
|
|
Pierre-Yves Chibon |
9c2953 |
with os.fdopen(
|
|
Pierre-Yves Chibon |
9c2953 |
os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb"
|
|
Pierre-Yves Chibon |
9c2953 |
) as stream:
|
|
Pierre-Yves Chibon |
893d4f |
stream.write(private_pem)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
public_key = private_key.public_key()
|
|
Pierre-Yves Chibon |
893d4f |
public_pem = _serialize_public_ssh_key(public_key)
|
|
Pierre-Yves Chibon |
893d4f |
if public_pem:
|
|
Pierre-Yves Chibon |
9c2953 |
with open(keyfile + ".pub", "wb") as stream:
|
|
Pierre-Yves Chibon |
893d4f |
stream.write(public_pem)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
|
|
Pierre-Yves Chibon |
893d4f |
@pagure_task
|
|
Pierre-Yves Chibon |
893d4f |
def setup_mirroring(self, session, username, namespace, name):
|
|
Pierre-Yves Chibon |
9c2953 |
""" Setup the specified project for mirroring.
|
|
Pierre-Yves Chibon |
9c2953 |
"""
|
|
Pierre-Yves Chibon |
9c2953 |
plugin = pagure.lib.plugins.get_plugin("Mirroring")
|
|
Pierre-Yves Chibon |
893d4f |
plugin.db_object()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
930073 |
project = pagure.lib.query._get_project(
|
|
Pierre-Yves Chibon |
9c2953 |
session, namespace=namespace, name=name, user=username
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
public_key_name = werkzeug.secure_filename(project.fullname)
|
|
Pierre-Yves Chibon |
9c2953 |
ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
if not os.path.exists(ssh_folder):
|
|
Pierre-Yves Chibon |
893d4f |
os.makedirs(ssh_folder, mode=0o700)
|
|
Pierre-Yves Chibon |
893d4f |
else:
|
|
Pierre-Yves Chibon |
893d4f |
if os.path.islink(ssh_folder):
|
|
Pierre-Yves Chibon |
9c2953 |
raise pagure.exceptions.PagureException("SSH folder is a link")
|
|
Pierre-Yves Chibon |
893d4f |
folder_stat = os.stat(ssh_folder)
|
|
Pierre-Yves Chibon |
893d4f |
filemode = stat.S_IMODE(folder_stat.st_mode)
|
|
Pierre-Yves Chibon |
9c2953 |
if filemode != int("0700", 8):
|
|
Pierre-Yves Chibon |
893d4f |
raise pagure.exceptions.PagureException(
|
|
Pierre-Yves Chibon |
9c2953 |
"SSH folder had invalid permissions"
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
9c2953 |
if (
|
|
Pierre-Yves Chibon |
9c2953 |
folder_stat.st_uid != os.getuid()
|
|
Pierre-Yves Chibon |
9c2953 |
or folder_stat.st_gid != os.getgid()
|
|
Pierre-Yves Chibon |
9c2953 |
):
|
|
Pierre-Yves Chibon |
893d4f |
raise pagure.exceptions.PagureException(
|
|
Pierre-Yves Chibon |
9c2953 |
"SSH folder does not belong to the user or group running "
|
|
Pierre-Yves Chibon |
9c2953 |
"this task"
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
public_key_file = os.path.join(ssh_folder, "%s.pub" % public_key_name)
|
|
Pierre-Yves Chibon |
9c2953 |
_log.info("Public key of interest: %s", public_key_file)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
if os.path.exists(public_key_file):
|
|
Pierre-Yves Chibon |
9c2953 |
raise pagure.exceptions.PagureException("SSH key already exists")
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
_log.info("Creating public key")
|
|
Pierre-Yves Chibon |
893d4f |
_create_ssh_key(os.path.join(ssh_folder, public_key_name))
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
with open(public_key_file) as stream:
|
|
Pierre-Yves Chibon |
893d4f |
public_key = stream.read()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
if project.mirror_hook.public_key != public_key:
|
|
Pierre-Yves Chibon |
9c2953 |
_log.info("Updating information in the DB")
|
|
Pierre-Yves Chibon |
893d4f |
project.mirror_hook.public_key = public_key
|
|
Pierre-Yves Chibon |
893d4f |
session.add(project.mirror_hook)
|
|
Pierre-Yves Chibon |
893d4f |
session.commit()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
|
|
Pierre-Yves Chibon |
893d4f |
@pagure_task
|
|
Pierre-Yves Chibon |
893d4f |
def teardown_mirroring(self, session, username, namespace, name):
|
|
Pierre-Yves Chibon |
9c2953 |
""" Stop the mirroring of the specified project.
|
|
Pierre-Yves Chibon |
9c2953 |
"""
|
|
Pierre-Yves Chibon |
9c2953 |
plugin = pagure.lib.plugins.get_plugin("Mirroring")
|
|
Pierre-Yves Chibon |
893d4f |
plugin.db_object()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
930073 |
project = pagure.lib.query._get_project(
|
|
Pierre-Yves Chibon |
9c2953 |
session, namespace=namespace, name=name, user=username
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
public_key_name = werkzeug.secure_filename(project.fullname)
|
|
Pierre-Yves Chibon |
893d4f |
private_key_file = os.path.join(ssh_folder, public_key_name)
|
|
Pierre-Yves Chibon |
9c2953 |
public_key_file = os.path.join(ssh_folder, "%s.pub" % public_key_name)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
if os.path.exists(private_key_file):
|
|
Pierre-Yves Chibon |
893d4f |
os.unlink(private_key_file)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
if os.path.exists(public_key_file):
|
|
Pierre-Yves Chibon |
893d4f |
os.unlink(public_key_file)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
project.mirror_hook.public_key = None
|
|
Pierre-Yves Chibon |
893d4f |
session.add(project.mirror_hook)
|
|
Pierre-Yves Chibon |
893d4f |
session.commit()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
@conn.task(queue=pagure_config["MIRRORING_QUEUE"], bind=True)
|
|
Pierre-Yves Chibon |
893d4f |
@pagure_task
|
|
Pierre-Yves Chibon |
893d4f |
def mirror_project(self, session, username, namespace, name):
|
|
Pierre-Yves Chibon |
9c2953 |
""" Does the actual mirroring of the specified project.
|
|
Pierre-Yves Chibon |
9c2953 |
"""
|
|
Pierre-Yves Chibon |
9c2953 |
plugin = pagure.lib.plugins.get_plugin("Mirroring")
|
|
Pierre-Yves Chibon |
893d4f |
plugin.db_object()
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
930073 |
project = pagure.lib.query._get_project(
|
|
Pierre-Yves Chibon |
9c2953 |
session, namespace=namespace, name=name, user=username
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
repofolder = pagure_config["GIT_FOLDER"]
|
|
Pierre-Yves Chibon |
893d4f |
repopath = os.path.join(repofolder, project.path)
|
|
Pierre-Yves Chibon |
893d4f |
if not os.path.exists(repopath):
|
|
Pierre-Yves Chibon |
d6adba |
_log.warning("Git folder not found at: %s, bailing", repopath)
|
|
Pierre-Yves Chibon |
893d4f |
return
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
9c2953 |
ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"]
|
|
Pierre-Yves Chibon |
893d4f |
public_key_name = werkzeug.secure_filename(project.fullname)
|
|
Pierre-Yves Chibon |
893d4f |
private_key_file = os.path.join(ssh_folder, public_key_name)
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
996c7e |
if not os.path.exists(private_key_file):
|
|
Pierre-Yves Chibon |
d6adba |
_log.warning("No %s key found, bailing", private_key_file)
|
|
Pierre-Yves Chibon |
996c7e |
project.mirror_hook.last_log = "Private key not found on disk, bailing"
|
|
Pierre-Yves Chibon |
996c7e |
session.add(project.mirror_hook)
|
|
Pierre-Yves Chibon |
996c7e |
session.commit()
|
|
Pierre-Yves Chibon |
996c7e |
return
|
|
Pierre-Yves Chibon |
996c7e |
|
|
Pierre-Yves Chibon |
605eaa |
# Add the utility script allowing this feature to work on old(er) git.
|
|
Pierre-Yves Chibon |
605eaa |
here = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
|
Pierre-Yves Chibon |
605eaa |
script_file = os.path.join(here, "ssh_script.sh")
|
|
Pierre-Yves Chibon |
605eaa |
|
|
Pierre-Yves Chibon |
893d4f |
# Get the list of remotes
|
|
Pierre-Yves Chibon |
893d4f |
remotes = [
|
|
Pierre-Yves Chibon |
893d4f |
remote.strip()
|
|
Pierre-Yves Chibon |
9c2953 |
for remote in project.mirror_hook.target.split("\n")
|
|
Pierre-Yves Chibon |
9c2953 |
if project.mirror_hook
|
|
Pierre-Yves Chibon |
9c2953 |
and remote.strip()
|
|
Pierre-Yves Chibon |
893d4f |
and ssh_urlpattern.match(remote.strip())
|
|
Pierre-Yves Chibon |
893d4f |
]
|
|
Pierre-Yves Chibon |
893d4f |
|
|
Pierre-Yves Chibon |
893d4f |
# Push
|
|
Pierre-Yves Chibon |
893d4f |
logs = []
|
|
Pierre-Yves Chibon |
605eaa |
for remote in remotes:
|
|
Pierre-Yves Chibon |
893d4f |
_log.info(
|
|
Pierre-Yves Chibon |
605eaa |
"Pushing to remote %s using key: %s", remote, private_key_file
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
(stdout, stderr) = pagure.lib.git.read_git_lines(
|
|
Pierre-Yves Chibon |
605eaa |
["push", "--mirror", remote],
|
|
Pierre-Yves Chibon |
605eaa |
abspath=repopath,
|
|
Pierre-Yves Chibon |
9c2953 |
error=True,
|
|
Pierre-Yves Chibon |
605eaa |
env={"SSHKEY": private_key_file, "GIT_SSH": script_file},
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
log = "Output from the push:\n stdout: %s\n stderr: %s" % (
|
|
Pierre-Yves Chibon |
9c2953 |
stdout,
|
|
Pierre-Yves Chibon |
9c2953 |
stderr,
|
|
Pierre-Yves Chibon |
9c2953 |
)
|
|
Pierre-Yves Chibon |
893d4f |
logs.append(log)
|
|
Pierre-Yves Chibon |
605eaa |
|
|
Pierre-Yves Chibon |
893d4f |
if logs:
|
|
Pierre-Yves Chibon |
9c2953 |
project.mirror_hook.last_log = "\n".join(logs)
|
|
Pierre-Yves Chibon |
893d4f |
session.add(project.mirror_hook)
|
|
Pierre-Yves Chibon |
893d4f |
session.commit()
|
|
Pierre-Yves Chibon |
9c2953 |
_log.info("\n".join(logs))
|