From 8315535b75f1aeb312ea75422be10e947b339111 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: May 03 2018 08:30:30 +0000 Subject: Port pagure to python3 --- diff --git a/dev-data.py b/dev-data.py index 0827978..6589627 100644 --- a/dev-data.py +++ b/dev-data.py @@ -8,8 +8,8 @@ import sys from sqlalchemy import create_engine, MetaData import pagure -import pagure.flask_app import tests +from pagure.lib import create_session ''' Usage: @@ -19,7 +19,8 @@ python dev-data.py --populate python dev-data.py --all ''' -_config = pagure.config.config.reload_config() +_config = pagure.config.reload_config() + def init_database(): DB_URL = _config['DB_URL'] @@ -447,7 +448,7 @@ if __name__ == "__main__": empty_dev_db(meta, eng) if args.populate or args.all: - session = pagure.flask_app.SESSION + session = create_session(_config['DB_URL']) invalid_option = ['pingou', 'bar@pingou.com', 'foo', 'foo@bar.com'] print("") user_name = raw_input( diff --git a/files/api_key_expire_mail.py b/files/api_key_expire_mail.py index 350c5fd..cabaf9b 100644 --- a/files/api_key_expire_mail.py +++ b/files/api_key_expire_mail.py @@ -14,8 +14,8 @@ if 'PAGURE_CONFIG' not in os.environ \ os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' import pagure -from pagure import SESSION -from pagure.lib import model +import pagure.config +from pagure.lib import model, create_session def main(debug=False): @@ -27,6 +27,7 @@ def main(debug=False): email_dates = [email_day.date() for email_day in \ [current_time + timedelta(days=i) for i in day_diff_for_mail]] + SESSION = create_session(pagure.config.config['DB_URL']) tokens = SESSION.query(model.Token).all() for token in tokens: diff --git a/pagure-ev/pagure_stream_server.py b/pagure-ev/pagure_stream_server.py index 688799f..3f6f940 100644 --- a/pagure-ev/pagure_stream_server.py +++ b/pagure-ev/pagure_stream_server.py @@ -21,17 +21,19 @@ nc localhost 8080 import logging import os -import urlparse + import redis import trollius +from six.moves.urllib.parse import urlparse + log = logging.getLogger(__name__) if 'PAGURE_CONFIG' not in os.environ \ and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' + print('Using configuration file `/etc/pagure/pagure.cfg`') os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' @@ -50,7 +52,7 @@ POOL = redis.ConnectionPool( def _get_session(): global SESSION if SESSION is None: - print pagure.config.config['DB_URL'] + print(pagure.config.config['DB_URL']) SESSION = pagure.lib.create_session(pagure.config.config['DB_URL']) return SESSION @@ -203,7 +205,7 @@ def handle_client(client_reader, client_writer): log.warning("Invalid URL provided: %s" % data[1]) return - url = urlparse.urlsplit(data[1]) + url = urlparse(data[1]) try: obj = get_obj_from_path(url.path) diff --git a/pagure-milters/comment_email_milter.py b/pagure-milters/comment_email_milter.py index acb0818..7e6a706 100644 --- a/pagure-milters/comment_email_milter.py +++ b/pagure-milters/comment_email_milter.py @@ -10,9 +10,9 @@ import base64 import email import hashlib import os -import StringIO import sys import time +from io import BytesIO from multiprocessing import Process as Thread, Queue import Milter @@ -29,7 +29,7 @@ if 'PAGURE_CONFIG' not in os.environ \ logq = Queue(maxsize=4) -_config = pagure.config.config.reload_config() +_config = pagure.config.reload_config() def get_email_body(emailobj): @@ -90,7 +90,7 @@ class PagureMilter(Milter.Base): # NOTE: self.fp is only an *internal* copy of message data. You # must use addheader, chgheader, replacebody to change the message # on the MTA. - self.fp = StringIO.StringIO() + self.fp = BytesIO() self.canon_from = '@'.join(parse_addr(mailfrom)) self.fp.write('From %s %s\n' % (self.canon_from, time.ctime())) return Milter.CONTINUE diff --git a/pagure/api/fork.py b/pagure/api/fork.py index fa4c74b..bfd0068 100644 --- a/pagure/api/fork.py +++ b/pagure/api/fork.py @@ -8,6 +8,8 @@ """ +from __future__ import unicode_literals + import logging import os @@ -999,12 +1001,12 @@ def api_pull_request_create(repo, username=None, namespace=None): if not branch_to: raise pagure.exceptions.APIError( 400, error_code=APIERROR.EINVALIDREQ, - errors={u'branch_to': [u'This field is required.']}) + errors={'branch_to': ['This field is required.']}) branch_from = flask.request.form.get('branch_from') if not branch_from: raise pagure.exceptions.APIError( 400, error_code=APIERROR.EINVALIDREQ, - errors={u'branch_from': [u'This field is required.']}) + errors={'branch_from': ['This field is required.']}) parent = repo if repo.parent: diff --git a/pagure/cli/admin.py b/pagure/cli/admin.py index 8447169..bf3eb2f 100644 --- a/pagure/cli/admin.py +++ b/pagure/cli/admin.py @@ -304,13 +304,13 @@ def parse_arguments(args=None): def _ask_confirmation(): ''' Ask to confirm an action. ''' - action = raw_input('Do you want to continue? [y/N]') + action = input('Do you want to continue? [y/N]') return action.lower() in ['y', 'yes'] def _get_input(text): ''' Ask the user for input. ''' - return raw_input(text) + return input(text) def _get_project(arg_project, user=None): diff --git a/pagure/doc_utils.py b/pagure/doc_utils.py index 9d2ada6..047a18b 100644 --- a/pagure/doc_utils.py +++ b/pagure/doc_utils.py @@ -120,9 +120,7 @@ def convert_readme(content, ext, view_file_url=None): def load_doc(endpoint): """ Utility to load an RST file and turn it into fancy HTML. """ - rst = unicode(textwrap.dedent(endpoint.__doc__)) - - rst = modify_rst(rst) + rst = modify_rst(textwrap.dedent(endpoint.__doc__)) api_docs = docutils.examples.html_body(rst) diff --git a/pagure/docs_server.py b/pagure/docs_server.py index e3ea53b..e531c94 100644 --- a/pagure/docs_server.py +++ b/pagure/docs_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ - (c) 2014-2015 - Copyright Red Hat Inc + (c) 2014-2016 - Copyright Red Hat Inc Authors: Pierre-Yves Chibon @@ -179,7 +179,7 @@ def view_docs(repo, username=None, namespace=None, filename=None): (tree, content, filename) = __get_tree_and_content( repo_obj, commit, path) except pagure.exceptions.FileNotFoundException as err: - flask.flash(err.message, 'error') + flask.flash('%s' % err, 'error') except Exception as err: _log.exception(err) flask.abort(500, 'Unkown error encountered and reported') diff --git a/pagure/flask_app.py b/pagure/flask_app.py index ddb1e98..c2d713e 100644 --- a/pagure/flask_app.py +++ b/pagure/flask_app.py @@ -424,4 +424,4 @@ def _get_user(username): try: return pagure.lib.get_user(flask.g.session, username) except pagure.exceptions.PagureException as e: - flask.abort(404, e.message) + flask.abort(404, '%s' % e) diff --git a/pagure/hooks/__init__.py b/pagure/hooks/__init__.py index d397b51..1a22b9c 100644 --- a/pagure/hooks/__init__.py +++ b/pagure/hooks/__init__.py @@ -9,6 +9,8 @@ """ import os + +import six import wtforms from pagure.config import config as pagure_config @@ -22,7 +24,7 @@ class RequiredIf(wtforms.validators.Required): """ def __init__(self, fields, *args, **kwargs): - if isinstance(fields, basestring): + if isinstance(fields, six.string_types): fields = [fields] self.fields = fields super(RequiredIf, self).__init__(*args, **kwargs) diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index 519d1e8..97a0e11 100755 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -16,6 +16,7 @@ if 'PAGURE_CONFIG' not in os.environ \ import pygit2 # noqa: E402 +import six # noqa: E402 import pagure # noqa: E402 import pagure.flask_app # noqa: E402 @@ -76,7 +77,7 @@ def send_notifications(session, project, refname, revs, forced): authors = [] for author in auths: - if isinstance(author, basestring): + if isinstance(author, six.string_types): author = author else: author = author.to_json(public=True) @@ -96,7 +97,7 @@ def send_notifications(session, project, refname, revs, forced): authors=list(authors), agent=os.environ['GL_USER'], repo=project.to_json(public=True) - if not isinstance(project, basestring) else project, + if not isinstance(project, six.string_types) else project, ) fedmsg_hook = pagure.lib.plugins.get_plugin('Fedmsg') diff --git a/pagure/hooks/files/git_multimail.py b/pagure/hooks/files/git_multimail.py index 9bc2d02..7d0e1c6 100755 --- a/pagure/hooks/files/git_multimail.py +++ b/pagure/hooks/files/git_multimail.py @@ -58,6 +58,8 @@ import optparse import smtplib import time +import six + try: from email.utils import make_msgid from email.utils import getaddresses @@ -563,7 +565,7 @@ class GitObject(object): if not self.sha1: raise ValueError('Empty commit has no summary') - return iter(generate_summaries('--no-walk', self.sha1)).next() + return next(iter(generate_summaries('--no-walk', self.sha1))) def __eq__(self, other): return isinstance(other, GitObject) and self.sha1 == other.sha1 @@ -571,7 +573,7 @@ class GitObject(object): def __hash__(self): return hash(self.sha1) - def __nonzero__(self): + def __bool__(self): return bool(self.sha1) def __str__(self): @@ -1459,7 +1461,7 @@ class SMTPMailer(Mailer): try: msg = ''.join(lines) # turn comma-separated list into Python list if needed. - if isinstance(to_addrs, basestring): + if isinstance(to_addrs, six.string_types): to_addrs = [ email for (name, email) in getaddresses([to_addrs])] self.smtp.sendmail(self.envelopesender, to_addrs, msg) @@ -2148,7 +2150,7 @@ class IncrementalDateTime(object): def __init__(self): self.time = time.time() - def next(self): + def __next__(self): formatted = formatdate(self.time, True) self.time += 1 return formatted @@ -2378,7 +2380,7 @@ class Push(object): sys.stderr.write( 'Sending notification emails to: %s\n' % ( change.recipients,)) - extra_values = {'send_date': send_date.next()} + extra_values = {'send_date': next(send_date)} mailer.send( change.generate_email(self, body_filter, extra_values), change.recipients, @@ -2406,7 +2408,7 @@ class Push(object): rev = Revision( change, GitObject(sha1), num=num+1, tot=len(sha1s)) if rev.recipients: - extra_values = {'send_date': send_date.next()} + extra_values = {'send_date': next(send_date)} mailer.send( rev.generate_email(self, body_filter, extra_values), rev.recipients, diff --git a/pagure/internal/__init__.py b/pagure/internal/__init__.py index 7b82449..94fe0c8 100644 --- a/pagure/internal/__init__.py +++ b/pagure/internal/__init__.py @@ -294,7 +294,7 @@ def get_pull_request_ready_branch(): parent_repo_obj = repo_obj branches = {} - if not repo_obj.is_empty and repo_obj.listall_branches() > 1: + if not repo_obj.is_empty and len(repo_obj.listall_branches()) > 0: if not parent_repo_obj.is_empty \ and not parent_repo_obj.head_is_unborn: try: @@ -575,7 +575,7 @@ def get_branches_head(): repo_obj = pygit2.Repository(repopath) branches = {} - if not repo_obj.is_empty and repo_obj.listall_branches() > 1: + if not repo_obj.is_empty and len(repo_obj.listall_branches()) > 1: for branchname in repo_obj.listall_branches(): branch = repo_obj.lookup_branch(branchname) branches[branchname] = branch.get_object().hex diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 39db198..0e5c012 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -29,7 +29,6 @@ import logging import os import tempfile import subprocess -import urlparse import uuid import markdown import werkzeug @@ -42,6 +41,8 @@ import redis import six import sqlalchemy import sqlalchemy.schema + +from six.moves.urllib_parse import urlparse, urlencode, parse_qsl from sqlalchemy import func from sqlalchemy import asc, desc from sqlalchemy.orm import aliased @@ -160,7 +161,14 @@ def get_next_id(session, projectid): model.PullRequest.project_id == projectid ) - nid = max([el[0] for el in query1.union(query2).all()]) or 0 + ids = [ + el[0] + for el in query1.union(query2).all() + if el[0] is not None + ] + nid = 0 + if ids: + nid = max(ids) return nid + 1 @@ -243,6 +251,7 @@ def is_valid_ssh_key(key): stdout, stderr = proc.communicate() if proc.returncode != 0: return False + stdout = stdout.decode("utf-8") if _is_valid_ssh_key_force_md5 is None: # We grab the "key ID" portion of the very first key to verify the @@ -359,7 +368,7 @@ def create_user_ssh_keys_on_disk(user, gitolite_keydir): os.mkdir(keyline_dir) keyfile = os.path.join(keyline_dir, '%s.pub' % user.user) with open(keyfile, 'w') as stream: - stream.write(keys[i].strip().encode('UTF-8')) + stream.write(keys[i].strip()) def add_issue_comment(session, issue, comment, user, ticketfolder, @@ -432,7 +441,7 @@ def add_tag_obj(session, obj, tags, user, gitfolder): ''' Add a tag to an object (either an issue or a project). ''' user_obj = get_user(session, user) - if isinstance(tags, basestring): + if isinstance(tags, six.string_types): tags = [tags] added_tags = [] @@ -848,7 +857,7 @@ def remove_tags_obj(session, obj, tags, gitfolder, user): ''' Removes the specified tag(s) of a given object. ''' user_obj = get_user(session, user) - if isinstance(tags, basestring): + if isinstance(tags, six.string_types): tags = [tags] removed_tags = [] @@ -2194,7 +2203,7 @@ def search_projects( ) # No filtering is done if private == username i.e if the owner of the # project is viewing the project - elif isinstance(private, basestring) and private != username: + elif isinstance(private, six.string_types) and private != username: # if we use the sqlalchemy.or_ in projects.filter, it will # fail to find any projects assuming there are no rows in # the ProjectUser table due to the way sqlalchemy creates @@ -2446,7 +2455,7 @@ def search_issues( model.Issue.priority == priority ) if tags is not None and tags != []: - if isinstance(tags, basestring): + if isinstance(tags, six.string_types): tags = [tags] notags = [] ytags = [] @@ -2541,7 +2550,7 @@ def search_issues( query = query.filter( model.Issue.private == False # noqa: E712 ) - elif isinstance(private, basestring): + elif isinstance(private, six.string_types): user2 = aliased(model.User) user4 = aliased(model.User) query = query.filter( @@ -2562,7 +2571,7 @@ def search_issues( if no_milestones and milestones is not None and milestones != []: # Asking for issues with no milestone or a specific milestone - if isinstance(milestones, basestring): + if isinstance(milestones, six.string_types): milestones = [milestones] query = query.filter( (model.Issue.milestone.is_(None)) | @@ -2575,7 +2584,7 @@ def search_issues( ) elif milestones is not None and milestones != []: # Asking for a single specific milestone - if isinstance(milestones, basestring): + if isinstance(milestones, six.string_types): milestones = [milestones] query = query.filter( model.Issue.milestone.in_(milestones) @@ -2916,7 +2925,7 @@ def add_attachment(repo, issue, attachmentfolder, user, filename, filestream): # Write file filestream.seek(0) - with open(filepath, 'w') as stream: + with open(filepath, 'wb') as stream: stream.write(filestream.read()) tasks.add_file_to_git.delay( @@ -3119,7 +3128,7 @@ def add_email_to_user(session, user, user_email): def update_user_ssh(session, user, ssh_key, keydir): ''' Set up a new user into the database or update its information. ''' - if isinstance(user, basestring): + if isinstance(user, six.string_types): user = get_user(session, user) user.public_ssh_key = ssh_key @@ -3134,7 +3143,7 @@ def avatar_url_from_email(email, size=64, default='retro', dns=False): """ Our own implementation since fas doesn't support this nicely yet. """ - if isinstance(email, unicode): + if not isinstance(email, six.string_types): email = email.encode('utf-8') if dns: # pragma: no cover @@ -3147,8 +3156,9 @@ def avatar_url_from_email(email, size=64, default='retro', dns=False): default=default, ) else: - import urllib - query = urllib.urlencode({'s': size, 'd': default}) + query = urlencode({'s': size, 'd': default}) + if six.PY3: + email = email.encode('utf-8') hashhex = hashlib.sha256(email).hexdigest() return "https://seccdn.libravatar.org/avatar/%s?%s" % ( hashhex, query) @@ -3159,7 +3169,7 @@ def update_tags(session, obj, tags, username, gitfolder): This object can be either an issue or a project. """ - if isinstance(tags, basestring): + if isinstance(tags, six.string_types): tags = [tags] toadd = set(tags) - set(obj.tags_text) @@ -3197,7 +3207,7 @@ def update_dependency_issue( """ Update the dependency of a specified issue (adding or removing them) """ - if isinstance(depends, basestring): + if isinstance(depends, six.string_types): depends = [depends] toadd = set(depends) - set(issue.depending_text) @@ -3252,7 +3262,7 @@ def update_blocked_issue( removing them) """ - if isinstance(blocks, basestring): + if isinstance(blocks, six.string_types): blocks = [blocks] toadd = set(blocks) - set(issue.blocking_text) @@ -3800,8 +3810,8 @@ def filter_img_src(name, value): if name in ('alt', 'height', 'width', 'class', 'data-src'): return True if name == 'src': - parsed = urlparse.urlparse(value) - return (not parsed.netloc) or parsed.netloc == urlparse.urlparse( + parsed = urlparse(value) + return (not parsed.netloc) or parsed.netloc == urlparse( pagure_config['APP_URL']).netloc return False @@ -4293,10 +4303,10 @@ def get_watch_list(session, obj): def save_report(session, repo, name, url, username): """ Save the report of issues based on the given URL of the project. """ - url_obj = urlparse.urlparse(url) + url_obj = urlparse(url) url = url_obj.geturl().replace(url_obj.query, '') query = {} - for k, v in urlparse.parse_qsl(url_obj.query): + for k, v in parse_qsl(url_obj.query): if k in query: if isinstance(query[k], list): query[k].append(v) @@ -4445,7 +4455,7 @@ def get_yearly_stats_user(session, user, date, tz='UTC'): # us a dict with the dates as keys and the number of times each # date occurs in the data as the values, we return its items as # a list of tuples - return Counter([event.date_tz(tz) for event in events]).items() + return list(Counter([event.date_tz(tz) for event in events]).items()) def get_user_activity_day(session, user, date, tz='UTC'): diff --git a/pagure/lib/git.py b/pagure/lib/git.py index d6afe98..3fa6f95 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -7,6 +7,7 @@ Pierre-Yves Chibon """ +from __future__ import print_function, unicode_literals # pylint: disable=too-many-branches # pylint: disable=too-many-arguments @@ -24,6 +25,7 @@ import tempfile import arrow import pygit2 +import six from sqlalchemy.exc import SQLAlchemyError from pygit2.remote import RemoteCollection @@ -82,7 +84,7 @@ def commit_to_patch(repo_obj, commits, diff_view=False): subject = '[PATCH %s/%s] %s' % ( cnt + 1, len(commits), subject) - patch.append(u"""From {commit} Mon Sep 17 00:00:00 2001 + patch.append("""From {commit} Mon Sep 17 00:00:00 2001 From: {author_name} <{author_email}> Date: {date} Subject: {subject} @@ -161,6 +163,15 @@ def _maybe_wait(result): pass +def _make_signature(name, email): + if six.PY2: + if isinstance(name, six.text_type): + name = name.encode("utf-8") + if isinstance(email, six.text_type): + email = email.encode("utf-8") + return pygit2.Signature(name=name, email=email) + + def _update_git(obj, repo, repofolder): """ Update the given issue in its git. @@ -226,7 +237,7 @@ def _update_git(obj, repo, repofolder): parents.append(parent) # Author/commiter will always be this one - author = pygit2.Signature(name='pagure', email='pagure') + author = _make_signature(name='pagure', email='pagure') # Actually commit new_repo.create_commit( @@ -304,7 +315,7 @@ def _clean_git(obj, repo, repofolder): parents.append(parent) # Author/commiter will always be this one - author = pygit2.Signature(name='pagure', email='pagure') + author = _make_signature(name='pagure', email='pagure') # Actually commit new_repo.create_commit( @@ -831,9 +842,9 @@ def _add_file_to_git(repo, issue, attachmentfolder, ticketfolder, user, parents.append(parent) # Author/commiter will always be this one - author = pygit2.Signature( - name=user.username.encode('utf-8'), - email=user.default_email.encode('utf-8') + author = _make_signature( + name=user.username, + email=user.default_email, ) # Actually commit @@ -895,7 +906,7 @@ def _update_file_in_git( index = new_repo.index # Write down what changed - with open(file_path, 'w') as stream: + with open(file_path, 'wb') as stream: stream.write(content.replace('\r', '').encode('utf-8')) # Retrieve the list of files that changed @@ -930,10 +941,7 @@ def _update_file_in_git( # Author/commiter will always be this one name = user.fullname or user.username - author = pygit2.Signature( - name=name.encode('utf-8'), - email=email.encode('utf-8') - ) + author = _make_signature(name=name, email=email) # Actually commit commit = new_repo.create_commit( @@ -987,11 +995,13 @@ def read_output(cmd, abspath, input=None, keepends=False, **kw): cwd=abspath, **kw) (out, err) = procs.communicate(input) + out = out.decode('utf-8') + err = err.decode('utf-8') retcode = procs.wait() if retcode: - print 'ERROR: %s =-- %s' % (cmd, retcode) - print out - print err + print('ERROR: %s =-- %s' % (cmd, retcode)) + print(out) + print(err) if not keepends: out = out.rstrip('\n\r') return out @@ -1287,9 +1297,9 @@ def merge_pull_request( tree = new_repo.index.write_tree() user_obj = pagure.lib.get_user(session, username) commitname = user_obj.fullname or user_obj.user - author = pygit2.Signature( - commitname.encode('utf-8'), - user_obj.default_email.encode('utf-8')) + author = _make_signature( + commitname, + user_obj.default_email) commit = new_repo.create_commit( 'refs/heads/%s' % request.branch, author, @@ -1334,9 +1344,9 @@ def merge_pull_request( _log.info(' Basing on: %s - %s', head.hex, repo_commit.oid.hex) user_obj = pagure.lib.get_user(session, username) commitname = user_obj.fullname or user_obj.user - author = pygit2.Signature( - commitname.encode('utf-8'), - user_obj.default_email.encode('utf-8')) + author = _make_signature( + commitname, + user_obj.default_email) commit = new_repo.create_commit( 'refs/heads/%s' % request.branch, author, @@ -1443,13 +1453,13 @@ def get_diff_info(repo_obj, orig_repo, branch_from, branch_to, prid=None): com = None if branch: try: - com = main_walker.next() + com = next(main_walker) main_commits.add(com.oid.hex) except StopIteration: com = None try: - branch_commit = branch_walker.next() + branch_commit = next(branch_walker) except StopIteration: branch_commit = None diff --git a/pagure/lib/git_auth.py b/pagure/lib/git_auth.py index 65d315b..233d1ea 100644 --- a/pagure/lib/git_auth.py +++ b/pagure/lib/git_auth.py @@ -10,7 +10,6 @@ from __future__ import print_function import abc -import gdbm import logging import os import pkg_resources @@ -18,6 +17,8 @@ import subprocess import tempfile import werkzeug +from six import with_metaclass +from six.moves import dbm_gnu import pagure.exceptions from pagure.config import config as pagure_config @@ -47,13 +48,11 @@ def get_git_auth_helper(backend, *args, **kwargs): return cls(*args, **kwargs) -class GitAuthHelper(object): +class GitAuthHelper(with_metaclass(abc.ABCMeta, object)): """ The class to inherit from when creating your own git authentication helper. """ - __metaclass__ = abc.ABCMeta - @classmethod @abc.abstractmethod def generate_acls(self, project, group=None): @@ -471,13 +470,13 @@ class Gitolite2Auth(GitAuthHelper): _log.info('Remove project from the gitolite cache file') cf = None try: - # unfortunately gdbm.open isn't a context manager in Python 2 :( - cf = gdbm.open(cache_file, 'ws') + # unfortunately dbm_gnu.open isn't a context manager in Python 2 :( + cf = dbm_gnu.open(cache_file, 'ws') for repo in ['', 'docs/', 'tickets/', 'requests/']: to_remove = repo + project.fullname if to_remove.encode('ascii') in cf: del cf[to_remove] - except gdbm.error as e: + except dbm_gnu.error as e: msg = ( 'Failed to remove project from gitolite cache: {msg}' .format(msg=e[1]) diff --git a/pagure/lib/login.py b/pagure/lib/login.py index a5b6c58..c8f117b 100644 --- a/pagure/lib/login.py +++ b/pagure/lib/login.py @@ -9,6 +9,8 @@ """ +from __future__ import unicode_literals + import random import string import hashlib @@ -59,8 +61,8 @@ def generate_hashed_value(password): if not isinstance(password, six.text_type): raise ValueError("Password supplied is not unicode text") - return '$2$' + bcrypt.hashpw(password.encode('utf-8'), - bcrypt.gensalt()) + return b'$2$' + bcrypt.hashpw(password.encode('utf-8'), + bcrypt.gensalt()) def check_password(entered_password, user_password, seed=None): @@ -69,31 +71,34 @@ def check_password(entered_password, user_password, seed=None): :arg entered_password: password entered by the user. :type entered_password: str (Python 3) or unicode (Python 2) :arg user_password: the hashed string fetched from the database. + :type user_password: bytes :return: a Boolean depending upon the entered_password, True if the password matches ''' if not isinstance(entered_password, six.text_type): raise ValueError("Entered password is not unicode text") + if isinstance(user_password, six.text_type): + raise ValueError("DB password is not bytes") - if not user_password.count('$') >= 2: + if not user_password.count(b'$') >= 2: raise pagure.exceptions.PagureException( 'Password of unknown version found in the database' ) - _, version, user_password = user_password.split('$', 2) + _, version, user_password = user_password.split(b'$', 2) - if version == '2': + if version == b'2': password = bcrypt.hashpw( entered_password.encode('utf-8'), - user_password.encode('utf-8')) - elif version == '1': + user_password) + elif version == b'1': password = '%s%s' % (entered_password, seed) - password = hashlib.sha512(password.encode('utf-8')).hexdigest() + password = hashlib.sha512( + password.encode('utf-8')).hexdigest().encode("utf-8") else: raise pagure.exceptions.PagureException( 'Password of unknown version found in the database' ) - return constant_time.bytes_eq( - password.encode('utf-8'), user_password.encode('utf-8')) + return constant_time.bytes_eq(password, user_password) diff --git a/pagure/lib/mimetype.py b/pagure/lib/mimetype.py index f70443b..149b906 100644 --- a/pagure/lib/mimetype.py +++ b/pagure/lib/mimetype.py @@ -1,7 +1,12 @@ # -*- coding: utf-8 -*- + +from __future__ import unicode_literals + import logging import mimetypes import kitchen.text.converters as ktc +import six + import pagure.lib.encoding_utils @@ -24,7 +29,7 @@ def guess_type(filename, data): mimetype, encoding = mimetypes.guess_type(filename) if data: if not mimetype: - if '\0' in data: + if not isinstance(data, six.text_type) and b'\0' in data: mimetype = 'application/octet-stream' else: mimetype = 'text/plain' diff --git a/pagure/lib/model.py b/pagure/lib/model.py index f7a8ac4..7571fee 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -8,6 +8,8 @@ """ +from __future__ import unicode_literals + __requires__ = ['SQLAlchemy >= 0.8', 'jinja2 >= 2.4'] # noqa import pkg_resources # noqa: E402,F401 @@ -134,7 +136,9 @@ def create_default_status(session, acls=None): session.rollback() _log.debug('Type %s could not be added', grptype) - for acl in sorted(acls) or {}: + acls = acls or {} + keys = sorted(list(acls.keys())) + for acl in keys: item = ACL( name=acl, description=acls[acl] @@ -1246,6 +1250,12 @@ class Issue(BASE): for comment in self.comments if not comment.notification] + @property + def sortable_priority(self): + ''' Return an empty string if no priority is set allowing issues to + be sorted using this attribute. ''' + return self.priority if self.priority else '' + def to_json(self, public=False, with_comments=True, with_project=False): ''' Returns a dictionary representation of the issue. diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index 60dcdb8..5097418 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -8,7 +8,8 @@ pagure notifications. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals + # pylint: disable=too-many-branches # pylint: disable=too-many-arguments @@ -18,12 +19,13 @@ import datetime import hashlib import json import logging -import urlparse import re import smtplib import time +import six from email.header import Header from email.mime.text import MIMEText +from six.moves.urllib_parse import urljoin import flask import pagure.lib @@ -280,6 +282,8 @@ def send_email(text, subject, to_mail, from_email = pagure_config.get( 'FROM_EMAIL', 'pagure@fedoraproject.org') + if isinstance(from_email, bytes): + from_email = from_email.decode('utf-8') if user_from: header = Header(user_from, 'utf-8') from_email = '%s <%s>' % (header, from_email) @@ -298,7 +302,7 @@ def send_email(text, subject, to_mail, smtp = None for mailto in to_mail.split(','): msg = MIMEText(text.encode('utf-8'), 'plain', 'utf-8') - msg['Subject'] = header = Header( + msg['Subject'] = Header( '[%s] %s' % (subject_tag, subject), 'utf-8') msg['From'] = from_email @@ -320,13 +324,17 @@ def send_email(text, subject, to_mail, # Send the message via our own SMTP server, but don't include the # envelope header. - if isinstance(mailto, unicode): - mailto = mailto.encode('utf-8') msg['To'] = mailto salt = pagure_config.get('SALT_EMAIL') - if isinstance(mail_id, unicode): - mail_id = mail_id.encode('utf-8') - mhash = hashlib.sha512('<%s>%s%s' % (mail_id, salt, mailto)) + if salt and not isinstance(salt, bytes): + salt = salt.encode('utf-8') + + key = (b'<' + mail_id.encode("utf-8") + b'>' + salt + + mailto.encode("utf-8")) + if six.PY3 and isinstance(key, str): + key = key.encode('utf-8') + mhash = hashlib.sha512(key) + msg['Reply-To'] = 'reply+%s@%s' % ( mhash.hexdigest(), pagure_config['DOMAIN_EMAIL_NOTIFICATIONS']) @@ -339,7 +347,7 @@ def send_email(text, subject, to_mail, _log.debug('in_reply_to: %s', in_reply_to) _log.debug('mail_id: %s', mail_id) _log.debug('Contents:') - _log.debug(text.encode('utf-8')) + _log.debug('%s' % text) _log.debug('*****************') _log.debug(msg.as_string()) _log.debug('*****/EMAIL******') @@ -377,7 +385,7 @@ def notify_new_comment(comment, user=None): to the issue. ''' - text = u""" + text = """ %s added a new comment to an issue you are following: `` %s @@ -415,7 +423,7 @@ def notify_new_issue(issue, user=None): ''' Notify the people following a project that a new issue was added to it. ''' - text = u""" + text = """ %s reported a new issue against the project: `%s` that you are following: `` %s @@ -452,7 +460,7 @@ def notify_assigned_issue(issue, new_assignee, user): action = 'reset' if new_assignee: action = 'assigned to `%s`' % new_assignee.user - text = u""" + text = """ The issue: `%s` of project: `%s` has been %s by %s. %s @@ -489,7 +497,7 @@ def notify_status_change_issue(issue, user): status = issue.status if status.lower() != 'open' and issue.close_status: status = '%s as %s' % (status, issue.close_status) - text = u""" + text = """ The status of the issue: `%s` of project: `%s` has been updated to: %s by %s. %s @@ -519,7 +527,7 @@ The status of the issue: `%s` of project: `%s` has been updated to: %s by %s. def notify_meta_change_issue(issue, user, msg): ''' Notify that a custom field changed ''' - text = u""" + text = """ `%s` updated issue. %s @@ -551,7 +559,7 @@ def notify_assigned_request(request, new_assignee, user): action = 'reset' if new_assignee: action = 'assigned to `%s`' % new_assignee.user - text = u""" + text = """ The pull-request: `%s` of project: `%s` has been %s by %s. %s @@ -586,7 +594,7 @@ def notify_new_pull_request(request): ''' Notify the people following a project that a new pull-request was added to it. ''' - text = u""" + text = """ %s opened a new pull-request against the project: `%s` that you are following: `` %s @@ -619,7 +627,7 @@ def notify_merge_pull_request(request, user): ''' Notify the people following a project that a pull-request was merged in it. ''' - text = u""" + text = """ %s merged a pull-request against the project: `%s` that you are following. Merged pull-request: @@ -655,7 +663,7 @@ def notify_cancelled_pull_request(request, user): ''' Notify the people following a project that a pull-request was cancelled in it. ''' - text = u""" + text = """ %s canceled a pull-request against the project: `%s` that you are following. Cancelled pull-request: @@ -691,7 +699,7 @@ def notify_pull_request_comment(comment, user): ''' Notify the people following a pull-request that a new comment was added to it. ''' - text = u""" + text = """ %s commented on the pull-request: `%s` that you are following: `` %s @@ -727,7 +735,7 @@ def notify_pull_request_flag(flag, user): ''' Notify the people following a pull-request that a new flag was added to it. ''' - text = u""" + text = """ %s flagged the pull-request `%s` as %s: %s %s @@ -760,12 +768,12 @@ def notify_new_email(email, user): root_url = pagure_config.get('APP_URL', flask.request.url_root) - url = urlparse.urljoin( + url = urljoin( root_url or flask.request.url_root, flask.url_for('ui_ns.confirm_email', token=email.token), ) - text = u"""Dear %(username)s, + text = """Dear %(username)s, You have registered a new email on pagure at %(root_url)s. @@ -797,11 +805,10 @@ def notify_new_commits(abspath, project, branch, commits): for commit in commits: commits_info.append({ 'commit': commit, - # we want these to be unicodes (read_output gives us str) 'author': pagure.lib.git.get_author( - commit, abspath).decode('utf-8'), + commit, abspath), 'subject': pagure.lib.git.get_commit_subject( - commit, abspath).decode('utf-8') + commit, abspath) }) # make sure this is unicode @@ -839,7 +846,7 @@ def notify_commit_flag(flag, user): ''' Notify the people following a project that a new flag was added to one of its commit. ''' - text = u""" + text = """ %s flagged the commit `%s` as %s: %s %s diff --git a/pagure/lib/repo.py b/pagure/lib/repo.py index 11ac301..d20e6fa 100644 --- a/pagure/lib/repo.py +++ b/pagure/lib/repo.py @@ -7,6 +7,7 @@ Pierre-Yves Chibon """ +from __future__ import print_function import logging import os @@ -111,7 +112,7 @@ class PagureRepo(pygit2.Repository): (out, err) = procs.communicate(line) retcode = procs.wait() if retcode: - print 'ERROR: %s =-- %s' % (cmd, retcode) - print out - print err + print('ERROR: %s =-- %s' % (cmd, retcode)) + print(out) + print(err) out = out.rstrip('\n\r') diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index c84ea61..ec39c2c 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -8,6 +8,8 @@ """ +from __future__ import unicode_literals + import collections import datetime import gc @@ -299,7 +301,7 @@ def create_project(self, session, username, namespace, name, add_readme, author = author.encode('utf-8') author_email = author_email.encode('utf-8') author = pygit2.Signature(author, author_email) - content = u"# %s\n\n%s" % (name, project.description) + content = "# %s\n\n%s" % (name, project.description) readme_file = os.path.join(temp_gitrepo.workdir, "README.md") with open(readme_file, 'wb') as stream: stream.write(content.encode('utf-8')) diff --git a/pagure/perfrepo.py b/pagure/perfrepo.py index ca0b54d..9c45ab5 100644 --- a/pagure/perfrepo.py +++ b/pagure/perfrepo.py @@ -7,12 +7,15 @@ Patrick Uiterwijk """ +from __future__ import print_function + import pprint import os import traceback import types +import six import pygit2 import _pygit2 @@ -65,10 +68,10 @@ class FakeWalker(object): return self - def next(self): + def __next__(self): STATS['walks'][self.wid]['steps'] += 1 TOTALS['steps'] += 1 - resp = self.parent.next() + resp = next(self.parent) return resp @@ -77,9 +80,9 @@ class FakeDiffHunk(object): self.parent = parent def __getattr__(self, attr): - print 'Getting Fake Hunk %s' % attr + print('Getting Fake Hunk %s' % attr) resp = getattr(self.parent, attr) - print 'Response: %s' % resp + print('Response: %s' % resp) return resp @@ -117,9 +120,9 @@ class FakeDiffer(object): self.iter = self.parent.__iter__() return self - def next(self): + def __next__(self): STATS['diffs'][self.did]['steps'] += 1 - resp = self.iter.next() + resp = next(self.iter) if isinstance(resp, _pygit2.Patch): resp = FakeDiffPatch(resp) else: @@ -130,12 +133,11 @@ class FakeDiffer(object): return len(self.parent) -class PerfRepo(object): +class PerfRepo(six.with_metaclass(PerfRepoMeta, object)): """ An utility class allowing to go around pygit2's inability to be stable. """ - __metaclass__ = PerfRepoMeta def __init__(self, path): STATS['repo_inits'].append((path, traceback.extract_stack(limit=2)[0])) @@ -185,13 +187,15 @@ class PerfRepo(object): self.iter = self.repo.__iter__() return self - def next(self): + def __next__(self): STATS['walks'][self.wid]['steps'] += 1 TOTALS['steps'] += 1 - return self.iter.next() + return next(self.iter) -pygit2.Repository = PerfRepo +if six.PY2: + # Disable perfrepo on PY3, it doesn't work + pygit2.Repository = PerfRepo def reset_stats(): @@ -215,7 +219,7 @@ def print_stats(response): if not os.environ.get('PAGURE_PERFREPO_VERBOSE'): return response - print 'Statistics:' + print('Statistics:') pprint.pprint(STATS) return response diff --git a/pagure/pfmarkdown.py b/pagure/pfmarkdown.py index 3eedb8c..a8a8111 100644 --- a/pagure/pfmarkdown.py +++ b/pagure/pfmarkdown.py @@ -26,6 +26,7 @@ import markdown.preprocessors import markdown.util import pygit2 import re +import six import pagure.lib from pagure.config import config as pagure_config @@ -444,7 +445,7 @@ def _obj_anchor_tag(user, namespace, repo, obj, text): :return: An element tree containing the href to the issue or PR :rtype: xml.etree.ElementTree.Element """ - if isinstance(obj, basestring): + if isinstance(obj, six.string_types): url = flask.url_for( 'ui_ns.view_commit', username=user, namespace=namespace, repo=repo, commitid=obj) diff --git a/pagure/templates/commit.html b/pagure/templates/commit.html index 86be630..f9459c1 100644 --- a/pagure/templates/commit.html +++ b/pagure/templates/commit.html @@ -100,7 +100,11 @@ {% endif %} {% endif %} - {{ patch.new_file_path | unicode }} + {% if patch | hasattr('new_file_path') %} + {{ patch.new_file_path | unicode }} + {% elif patch | hasattr('delta') %} + {{ patch.delta.new_file.path | unicode }} + {% endif %} {% endfor %} diff --git a/pagure/templates/pull_request.html b/pagure/templates/pull_request.html index c0bf088..cb345ff 100644 --- a/pagure/templates/pull_request.html +++ b/pagure/templates/pull_request.html @@ -498,10 +498,10 @@ {%- else -%} {% set patchtype = "moved"%} - {{lineschanged(True, True)}} + {{ lineschanged(True, True) }}
{{patch.delta.old_file.path}}
- {{viewfilelink(patch.delta.new_file.path)}} + {{ viewfilelink(patch.delta.new_file.path) }}
file moved
{%- endif -%} diff --git a/pagure/templates/roadmap.html b/pagure/templates/roadmap.html index d6ee572..2937a20 100644 --- a/pagure/templates/roadmap.html +++ b/pagure/templates/roadmap.html @@ -37,7 +37,7 @@ - {% for issue in issues |sort(attribute='priority') %} + {% for issue in issues |sort(attribute='sortable_priority') %} {% if status is none or status|lower == 'all' or issue.status == status %} diff --git a/pagure/ui/filters.py b/pagure/ui/filters.py index dcb430f..332424b 100644 --- a/pagure/ui/filters.py +++ b/pagure/ui/filters.py @@ -14,13 +14,13 @@ import textwrap -import urlparse import arrow import flask -import md5 +import hashlib import six +from six.moves.urllib.parse import urlparse, parse_qsl from pygments import highlight from pygments.lexers.text import DiffLexer from pygments.formatters import HtmlFormatter @@ -92,8 +92,8 @@ def format_loc(loc, commit=None, filename=None, tree_id=None, prequest=None, comments = {} if prequest and not isinstance(prequest, flask.wrappers.Request): for com in prequest.comments: - if commit and unicode(com.commit_id) == unicode(commit) \ - and unicode(com.filename) == unicode(filename): + if commit and com.commit_id == commit \ + and com.filename == filename: if com.line in comments: comments[com.line].append(com) else: @@ -110,6 +110,8 @@ def format_loc(loc, commit=None, filename=None, tree_id=None, prequest=None, if line == '': break if filename and commit: + if isinstance(filename, str) and six.PY2: + filename = filename.decode('UTF-8') output.append( '' ' 1: + and len(repo_obj.listall_branches()) > 1: if not orig_repo.head_is_unborn: compare_branch = orig_repo.lookup_branch( @@ -545,7 +546,7 @@ def view_file(repo, identifier, filename, username=None, namespace=None): '.gif', '.png', '.bmp', '.tif', '.tiff', '.jpg', '.jpeg', '.ppm', '.pnm', '.pbm', '.pgm', '.webp', '.ico'): try: - Image.open(StringIO(content.data)) + Image.open(BytesIO(content.data)) output_type = 'image' except IOError as err: _log.debug( @@ -701,6 +702,8 @@ def view_raw_file( # First commit in the repo diff = commit.tree.diff_to_tree(swap=True) data = diff.patch + if six.PY3: + data = data.encode('utf-8').decode('utf-8') if not data: flask.abort(404, 'No content found') @@ -2128,7 +2131,9 @@ def edit_file(repo, branchname, filename, username=None, namespace=None): flask.abort(400, 'Cannot edit binary files') else: - data = form.content.data.decode('utf-8') + data = form.content.data + if not isinstance(data, six.string_types): + data = data.decode('utf-8') return flask.render_template( 'edit_file.html', diff --git a/pagure/utils.py b/pagure/utils.py index 0c9222a..4997806 100644 --- a/pagure/utils.py +++ b/pagure/utils.py @@ -10,11 +10,12 @@ import os import re -import urlparse +from six.moves.urllib.parse import urlparse, urljoin from functools import wraps import flask import pygit2 +import six import werkzeug from pagure.config import config as pagure_config @@ -40,9 +41,8 @@ def is_safe_url(target): # pragma: no cover """ Checks that the target url is safe and sending to the current website not some other malicious one. """ - ref_url = urlparse.urlparse(flask.request.host_url) - test_url = urlparse.urlparse( - urlparse.urljoin(flask.request.host_url, target)) + ref_url = urlparse(flask.request.host_url) + test_url = urlparse(urljoin(flask.request.host_url, target)) return test_url.scheme in ('http', 'https') and \ ref_url.netloc == test_url.netloc @@ -183,7 +183,9 @@ def __get_file_in_tree(repo_obj, tree, filepath, bail_on_tree=False): if isinstance(tree, pygit2.Blob): return for entry in tree: - fname = entry.name.decode('utf-8') + fname = entry.name + if six.PY2: + fname = entry.name.decode('utf-8') if fname == filename: if len(filepath) == 1: blob = repo_obj.get(entry.id)