|
Patrick Uiterwijk |
a50651 |
# -*- coding: utf-8 -*-
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
(c) 2014-2018 - Copyright Red Hat Inc
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
Authors:
|
|
Patrick Uiterwijk |
a50651 |
Patrick Uiterwijk <puiterwijk@redhat.com></puiterwijk@redhat.com>
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
|
|
Pierre-Yves Chibon |
67d1cc |
from __future__ import unicode_literals, absolute_import
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
import logging
|
|
Patrick Uiterwijk |
a50651 |
import subprocess
|
|
Patrick Uiterwijk |
a50651 |
import tempfile
|
|
Patrick Uiterwijk |
a50651 |
import os
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
import flask
|
|
Patrick Uiterwijk |
a50651 |
import requests
|
|
Patrick Uiterwijk |
a50651 |
import werkzeug
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
import pagure.exceptions
|
|
Patrick Uiterwijk |
a50651 |
import pagure.lib.git
|
|
Patrick Uiterwijk |
a50651 |
import pagure.lib.mimetype
|
|
Patrick Uiterwijk |
a50651 |
import pagure.lib.plugins
|
|
Pierre-Yves Chibon |
930073 |
import pagure.lib.query
|
|
Patrick Uiterwijk |
a50651 |
import pagure.lib.tasks
|
|
Patrick Uiterwijk |
a50651 |
import pagure.forms
|
|
Patrick Uiterwijk |
a50651 |
import pagure.ui.plugins
|
|
Patrick Uiterwijk |
a50651 |
from pagure.config import config as pagure_config
|
|
Patrick Uiterwijk |
a50651 |
from pagure.ui import UI_NS
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
_log = logging.getLogger(__name__)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
def proxy_raw_git():
|
|
Patrick Uiterwijk |
a50651 |
""" Proxy a request to Git or gitolite3 via a subprocess.
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
This should get called after it is determined the requested project
|
|
Patrick Uiterwijk |
a50651 |
is not on repoSpanner.
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
# We are going to shell out to gitolite-shell. Prepare the env it needs.
|
|
Patrick Uiterwijk |
a50651 |
gitenv = {
|
|
Patrick Uiterwijk |
a3c93a |
"PATH": os.environ["PATH"],
|
|
Patrick Uiterwijk |
a50651 |
# These are the vars git-http-backend needs
|
|
Patrick Uiterwijk |
a50651 |
"PATH_INFO": flask.request.path,
|
|
Patrick Uiterwijk |
a50651 |
"REMOTE_USER": flask.request.remote_user,
|
|
Patrick Uiterwijk |
a50651 |
"REMOTE_ADDR": flask.request.remote_addr,
|
|
Patrick Uiterwijk |
a50651 |
"CONTENT_TYPE": flask.request.content_type,
|
|
Patrick Uiterwijk |
a50651 |
"QUERY_STRING": flask.request.query_string,
|
|
Patrick Uiterwijk |
a50651 |
"REQUEST_METHOD": flask.request.method,
|
|
Patrick Uiterwijk |
a50651 |
"GIT_PROJECT_ROOT": pagure_config["GIT_FOLDER"],
|
|
Patrick Uiterwijk |
a50651 |
# We perform access checks, so can bypass that of Git
|
|
Patrick Uiterwijk |
a50651 |
"GIT_HTTP_EXPORT_ALL": "true",
|
|
Patrick Uiterwijk |
a50651 |
# This might be needed by hooks
|
|
Patrick Uiterwijk |
a50651 |
"PAGURE_CONFIG": os.environ.get("PAGURE_CONFIG"),
|
|
Patrick Uiterwijk |
a50651 |
"PYTHONPATH": os.environ.get("PYTHONPATH"),
|
|
Patrick Uiterwijk |
a3c93a |
# Some HTTP headers that we want to pass through because they
|
|
Patrick Uiterwijk |
a3c93a |
# impact the request/response. Only add headers here that are
|
|
Patrick Uiterwijk |
a3c93a |
# "safe", as in they don't allow for other issues.
|
|
Patrick Uiterwijk |
a3c93a |
"HTTP_CONTENT_ENCODING": flask.request.content_encoding,
|
|
Patrick Uiterwijk |
a50651 |
}
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
gitolite = pagure_config["HTTP_REPO_ACCESS_GITOLITE"]
|
|
Patrick Uiterwijk |
a50651 |
if gitolite:
|
|
Patrick Uiterwijk |
a50651 |
gitenv.update(
|
|
Patrick Uiterwijk |
a50651 |
{
|
|
Patrick Uiterwijk |
a50651 |
# These are the additional vars gitolite needs
|
|
Patrick Uiterwijk |
a50651 |
# Fun fact: REQUEST_URI is not even mentioned in RFC3875
|
|
Patrick Uiterwijk |
a50651 |
"REQUEST_URI": flask.request.full_path,
|
|
Patrick Uiterwijk |
a50651 |
"GITOLITE_HTTP_HOME": pagure_config["GITOLITE_HOME"],
|
|
Patrick Uiterwijk |
a50651 |
"HOME": pagure_config["GITOLITE_HOME"],
|
|
Patrick Uiterwijk |
a50651 |
}
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
elif flask.request.remote_user:
|
|
Patrick Uiterwijk |
a50651 |
gitenv.update({"GL_USER": flask.request.remote_user})
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
# These keys are optional
|
|
Patrick Uiterwijk |
a50651 |
for key in (
|
|
Patrick Uiterwijk |
a50651 |
"REMOTE_USER",
|
|
Patrick Uiterwijk |
a50651 |
"REMOTE_ADDR",
|
|
Patrick Uiterwijk |
a50651 |
"CONTENT_TYPE",
|
|
Patrick Uiterwijk |
a50651 |
"QUERY_STRING",
|
|
Patrick Uiterwijk |
a50651 |
"PYTHONPATH",
|
|
Patrick Uiterwijk |
a3c93a |
"PATH",
|
|
Patrick Uiterwijk |
a3c93a |
"HTTP_CONTENT_ENCODING",
|
|
Patrick Uiterwijk |
a50651 |
):
|
|
Patrick Uiterwijk |
a50651 |
if not gitenv[key]:
|
|
Patrick Uiterwijk |
a50651 |
del gitenv[key]
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
for key in gitenv:
|
|
Patrick Uiterwijk |
a50651 |
if not gitenv[key]:
|
|
Patrick Uiterwijk |
a50651 |
raise ValueError("Value for key %s unknown" % key)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
if gitolite:
|
|
Patrick Uiterwijk |
a50651 |
cmd = [gitolite]
|
|
Patrick Uiterwijk |
a50651 |
else:
|
|
Patrick Uiterwijk |
a50651 |
cmd = ["/usr/bin/git", "http-backend"]
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
# Note: using a temporary files to buffer the input contents
|
|
Patrick Uiterwijk |
a50651 |
# is non-ideal, but it is a way to make sure we don't need to have
|
|
Patrick Uiterwijk |
a50651 |
# the full input (which can be very long) in memory.
|
|
Patrick Uiterwijk |
a50651 |
# Ideally, we'd directly stream, but that's an RFE for the future,
|
|
Patrick Uiterwijk |
a50651 |
# since that needs to happen in other threads so as to not block.
|
|
Patrick Uiterwijk |
a50651 |
# (See the warnings in the subprocess module)
|
|
Patrick Uiterwijk |
a50651 |
with tempfile.SpooledTemporaryFile() as infile:
|
|
Patrick Uiterwijk |
a50651 |
while True:
|
|
Patrick Uiterwijk |
a50651 |
block = flask.request.stream.read(4096)
|
|
Patrick Uiterwijk |
a50651 |
if not block:
|
|
Patrick Uiterwijk |
a50651 |
break
|
|
Patrick Uiterwijk |
a50651 |
infile.write(block)
|
|
Patrick Uiterwijk |
a50651 |
infile.seek(0)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
proc = subprocess.Popen(
|
|
Patrick Uiterwijk |
a50651 |
cmd, stdin=infile, stdout=subprocess.PIPE, stderr=None, env=gitenv
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
out = proc.stdout
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
# First, gather the response head
|
|
Patrick Uiterwijk |
a50651 |
headers = {}
|
|
Patrick Uiterwijk |
a50651 |
while True:
|
|
Patrick Uiterwijk |
a50651 |
line = out.readline()
|
|
Patrick Uiterwijk |
a50651 |
if not line:
|
|
Patrick Uiterwijk |
a50651 |
raise Exception("End of file while reading headers?")
|
|
Patrick Uiterwijk |
a50651 |
# This strips the \n, meaning end-of-headers
|
|
Patrick Uiterwijk |
a50651 |
line = line.strip()
|
|
Patrick Uiterwijk |
a50651 |
if not line:
|
|
Patrick Uiterwijk |
a50651 |
break
|
|
Patrick Uiterwijk |
a50651 |
header = line.split(b": ", 1)
|
|
Patrick Uiterwijk |
a50651 |
headers[header[0].lower()] = header[1]
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
if len(headers) == 0:
|
|
Patrick Uiterwijk |
a50651 |
raise Exception("No response at all received")
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
if "status" not in headers:
|
|
Patrick Uiterwijk |
a50651 |
# If no status provided, assume 200 OK as per RFC3875
|
|
Patrick Uiterwijk |
a50651 |
headers["status"] = "200 OK"
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
respcode, respmsg = headers.pop("status").split(" ", 1)
|
|
Patrick Uiterwijk |
a50651 |
wrapout = werkzeug.wsgi.wrap_file(flask.request.environ, out)
|
|
Patrick Uiterwijk |
a50651 |
return flask.Response(
|
|
Patrick Uiterwijk |
a50651 |
wrapout,
|
|
Patrick Uiterwijk |
a50651 |
status=int(respcode),
|
|
Patrick Uiterwijk |
a50651 |
headers=headers,
|
|
Patrick Uiterwijk |
a50651 |
direct_passthrough=True,
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
def proxy_repospanner(project, service):
|
|
Patrick Uiterwijk |
a50651 |
""" Proxy a request to repoSpanner.
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
Args:
|
|
Patrick Uiterwijk |
a50651 |
project (model.Project): The project being accessed
|
|
Patrick Uiterwijk |
a50651 |
service (String): The service as indicated by ?Service= in /info/refs
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
oper = os.path.basename(flask.request.path)
|
|
Patrick Uiterwijk |
a50651 |
if oper == "refs":
|
|
Patrick Uiterwijk |
a50651 |
oper = "info/refs?service=%s" % service
|
|
Patrick Uiterwijk |
a50651 |
regionurl, regioninfo = project.repospanner_repo_info("main")
|
|
Patrick Uiterwijk |
a50651 |
url = "%s/%s" % (regionurl, oper)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
b76be0 |
# Older flask/werkzeug versions don't support both an input and output
|
|
Patrick Uiterwijk |
b76be0 |
# stream: this results in a blank upload.
|
|
Patrick Uiterwijk |
b76be0 |
# So, we optimize for the direction the majority of the data will likely
|
|
Patrick Uiterwijk |
b76be0 |
# flow.
|
|
Patrick Uiterwijk |
b76be0 |
streamargs = {}
|
|
Patrick Uiterwijk |
b76be0 |
if service == "git-receive-pack":
|
|
Patrick Uiterwijk |
b76be0 |
# This is a Push operation, optimize for data from the client
|
|
Patrick Uiterwijk |
b76be0 |
streamargs["data"] = flask.request.stream
|
|
Patrick Uiterwijk |
b76be0 |
streamargs["stream"] = False
|
|
Patrick Uiterwijk |
b76be0 |
else:
|
|
Patrick Uiterwijk |
b76be0 |
# This is a Pull operation, optimize for data from the server
|
|
Patrick Uiterwijk |
b76be0 |
streamargs["data"] = flask.request.data
|
|
Patrick Uiterwijk |
b76be0 |
streamargs["stream"] = True
|
|
Patrick Uiterwijk |
b76be0 |
|
|
Patrick Uiterwijk |
a50651 |
resp = requests.request(
|
|
Patrick Uiterwijk |
a50651 |
flask.request.method,
|
|
Patrick Uiterwijk |
a50651 |
url,
|
|
Patrick Uiterwijk |
a50651 |
verify=regioninfo["ca"],
|
|
Patrick Uiterwijk |
a50651 |
cert=(regioninfo["push_cert"]["cert"], regioninfo["push_cert"]["key"]),
|
|
Patrick Uiterwijk |
a50651 |
headers={
|
|
Patrick Uiterwijk |
f64282 |
"Content-Encoding": flask.request.content_encoding,
|
|
Patrick Uiterwijk |
a50651 |
"Content-Type": flask.request.content_type,
|
|
Patrick Uiterwijk |
a50651 |
"X-Extra-Username": flask.request.remote_user,
|
|
Patrick Uiterwijk |
a50651 |
"X-Extra-Repotype": "main",
|
|
Patrick Uiterwijk |
a50651 |
"X-Extra-project_name": project.name,
|
|
Patrick Uiterwijk |
a50651 |
"x-Extra-project_user": project.user if project.is_fork else "",
|
|
Patrick Uiterwijk |
a50651 |
"X-Extra-project_namespace": project.namespace,
|
|
Patrick Uiterwijk |
a50651 |
},
|
|
Patrick Uiterwijk |
b76be0 |
**streamargs
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
# Strip out any headers that cause problems
|
|
Patrick Uiterwijk |
a50651 |
for name in ("transfer-encoding",):
|
|
Patrick Uiterwijk |
071abd |
if name in resp.headers:
|
|
Patrick Uiterwijk |
071abd |
del resp.headers[name]
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
return flask.Response(
|
|
Patrick Uiterwijk |
a50651 |
resp.iter_content(chunk_size=128),
|
|
Patrick Uiterwijk |
a50651 |
status=resp.status_code,
|
|
Patrick Uiterwijk |
a50651 |
headers=dict(resp.headers),
|
|
Patrick Uiterwijk |
b76be0 |
direct_passthrough=True,
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
def clone_proxy(project, username=None, namespace=None):
|
|
Patrick Uiterwijk |
a50651 |
""" Proxy the /info/refs endpoint for HTTP pull/push.
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
Note that for the clone endpoints, it's very explicit that <repo> has been</repo>
|
|
Patrick Uiterwijk |
a50651 |
renamed to <project>, to avoid the automatic repo searching from flask_app.</project>
|
|
Patrick Uiterwijk |
a50651 |
This means that we have a chance to trust REMOTE_USER to verify the users'
|
|
Patrick Uiterwijk |
a50651 |
access to the attempted repository.
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
if not pagure_config["ALLOW_HTTP_PULL_PUSH"]:
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(403, description="HTTP pull/push is not allowed")
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
071abd |
service = None
|
|
Patrick Uiterwijk |
a50651 |
if flask.request.path.endswith("/info/refs"):
|
|
Patrick Uiterwijk |
a50651 |
service = flask.request.args.get("service")
|
|
Patrick Uiterwijk |
a50651 |
if not service:
|
|
Patrick Uiterwijk |
a50651 |
# This is a Git client older than 1.6.6, and it doesn't work with
|
|
Patrick Uiterwijk |
a50651 |
# the smart protocol. We do not support the old protocol via HTTP.
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(400, description="Please switch to newer Git client")
|
|
Patrick Uiterwijk |
a50651 |
if service not in ("git-upload-pack", "git-receive-pack"):
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(400, description="Unknown service requested")
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
if "git-receive-pack" in flask.request.full_path:
|
|
Patrick Uiterwijk |
a50651 |
if not pagure_config["ALLOW_HTTP_PUSH"]:
|
|
Patrick Uiterwijk |
a50651 |
# Pushing (git-receive-pack) over HTTP is not allowed
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(403, description="HTTP pushing disabled")
|
|
Patrick Uiterwijk |
a50651 |
if not flask.request.remote_user:
|
|
Patrick Uiterwijk |
a50651 |
# Anonymous pushing... nope
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(403, description="Unauthenticated push not allowed")
|
|
Patrick Uiterwijk |
a50651 |
|
|
Pierre-Yves Chibon |
930073 |
project = pagure.lib.query.get_authorized_project(
|
|
Patrick Uiterwijk |
a50651 |
flask.g.session,
|
|
Patrick Uiterwijk |
a50651 |
project,
|
|
Patrick Uiterwijk |
a50651 |
user=username,
|
|
Patrick Uiterwijk |
a50651 |
namespace=namespace,
|
|
Patrick Uiterwijk |
a50651 |
asuser=flask.request.remote_user,
|
|
Patrick Uiterwijk |
a50651 |
)
|
|
Patrick Uiterwijk |
a50651 |
if not project:
|
|
Pierre-Yves Chibon |
7d07b9 |
_log.info(
|
|
Pierre-Yves Chibon |
7d07b9 |
"%s could not find project: %s for user %s and namespace %s",
|
|
Pierre-Yves Chibon |
7d07b9 |
flask.request.remote_user,
|
|
Pierre-Yves Chibon |
7d07b9 |
project,
|
|
Pierre-Yves Chibon |
7d07b9 |
username,
|
|
Pierre-Yves Chibon |
7d07b9 |
namespace,
|
|
Pierre-Yves Chibon |
7d07b9 |
)
|
|
Pierre-Yves Chibon |
c6cc5c |
flask.abort(404, description="Project not found")
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
if project.is_on_repospanner:
|
|
Patrick Uiterwijk |
a50651 |
return proxy_repospanner(project, service)
|
|
Patrick Uiterwijk |
a50651 |
else:
|
|
Patrick Uiterwijk |
a50651 |
return proxy_raw_git()
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
def add_clone_proxy_cmds():
|
|
Patrick Uiterwijk |
a50651 |
""" This function adds flask routes for all possible clone paths.
|
|
Patrick Uiterwijk |
a50651 |
|
|
Patrick Uiterwijk |
a50651 |
This comes down to:
|
|
Patrick Uiterwijk |
a50651 |
/(fork/<username>/)(<namespace>/)<project>(.git)</project></namespace></username>
|
|
Patrick Uiterwijk |
a50651 |
with an operation following, where operation is one of:
|
|
Patrick Uiterwijk |
a50651 |
- /info/refs (generic)
|
|
Patrick Uiterwijk |
a50651 |
- /git-upload-pack (pull)
|
|
Patrick Uiterwijk |
a50651 |
- /git-receive-pack (push)
|
|
Patrick Uiterwijk |
a50651 |
"""
|
|
Patrick Uiterwijk |
a50651 |
for prefix in (
|
|
Patrick Uiterwijk |
a50651 |
"<project>",</project>
|
|
Patrick Uiterwijk |
a50651 |
"<namespace>/<project>",</project></namespace>
|
|
Patrick Uiterwijk |
a50651 |
"forks/<username>/<project>",</project></username>
|
|
Patrick Uiterwijk |
a50651 |
"forks/<username>/<namespace>/<project>",</project></namespace></username>
|
|
Patrick Uiterwijk |
a50651 |
):
|
|
Patrick Uiterwijk |
a50651 |
for suffix in ("", ".git"):
|
|
Patrick Uiterwijk |
a50651 |
for oper in ("info/refs", "git-receive-pack", "git-upload-pack"):
|
|
Patrick Uiterwijk |
a50651 |
route = "/%s%s/%s" % (prefix, suffix, oper)
|
|
Patrick Uiterwijk |
a50651 |
methods = ("GET",) if oper == "info/refs" else ("POST",)
|
|
Patrick Uiterwijk |
a50651 |
UI_NS.add_url_rule(
|
|
Patrick Uiterwijk |
a50651 |
route, view_func=clone_proxy, methods=methods
|
|
Patrick Uiterwijk |
a50651 |
)
|