Blame files/load_from_disk.py

Pierre-Yves Chibon e8bd26
#!/usr/bin/env python
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
import argparse
Pierre-Yves Chibon e8bd26
import requests
Pierre-Yves Chibon e8bd26
import os
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
from sqlalchemy.exc import SQLAlchemyError
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
if 'PAGURE_CONFIG' not in os.environ \
Pierre-Yves Chibon e8bd26
        and os.path.exists('/etc/pagure/pagure.cfg'):
Pierre-Yves Chibon e8bd26
    print 'Using configuration file `/etc/pagure/pagure.cfg`'
Pierre-Yves Chibon e8bd26
    os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
import pagure
Pierre-Yves Chibon 0a6f4e
import pagure.lib
Pierre-Yves Chibon 0a6f4e
import pagure.lib.model
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
def get_poc_of_pkgs(debug=False):
Pierre-Yves Chibon e8bd26
    """ Retrieve a dictionary giving the point of contact of each package
Pierre-Yves Chibon e8bd26
    in pkgdb.
Pierre-Yves Chibon e8bd26
    """
Pierre-Yves Chibon e8bd26
    if debug:
Pierre-Yves Chibon e8bd26
        print 'Querying pkgdb'
Pierre-Yves Chibon 0a6f4e
    PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/'
Pierre-Yves Chibon e8bd26
    req = requests.get(PKGDB_URL + 'bugzilla').text
Pierre-Yves Chibon e8bd26
    if debug:
Pierre-Yves Chibon e8bd26
        print 'Pkgdb data retrieved, getting POC'
Pierre-Yves Chibon e8bd26
    pkgs = {}
Pierre-Yves Chibon e8bd26
    for line in req.split('\n'):
Pierre-Yves Chibon e8bd26
        line = line.strip()
Pierre-Yves Chibon e8bd26
        if not line or line.startswith('#'):
Pierre-Yves Chibon e8bd26
            continue
Pierre-Yves Chibon e8bd26
        line = line.split('|')
Pierre-Yves Chibon e8bd26
        if len(line) < 4:
Pierre-Yves Chibon e8bd26
            continue
Pierre-Yves Chibon e8bd26
        pkgs[line[1]] = line[3]
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
    return pkgs
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
def main(folder, debug=False):
Pierre-Yves Chibon e8bd26
    """
Pierre-Yves Chibon e8bd26
    Logic:
Pierre-Yves Chibon e8bd26
    - Query the list of maintainer/PoC from pkgdb
Pierre-Yves Chibon e8bd26
    - Browse the directory
Pierre-Yves Chibon e8bd26
    - For each git in the directory, create the project with the correct POC
Pierre-Yves Chibon e8bd26
    """
Pierre-Yves Chibon e8bd26
    pocs = get_poc_of_pkgs(debug=debug)
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
    if debug:
Pierre-Yves Chibon e8bd26
        print 'Adding the user to the DB'
Pierre-Yves Chibon e8bd26
    for user in sorted(set(pocs.values())):
Pierre-Yves Chibon e8bd26
        if debug:
Pierre-Yves Chibon e8bd26
            print user
Pierre-Yves Chibon e8bd26
        try:
Pierre-Yves Chibon e8bd26
            pagure.lib.set_up_user(
Pierre-Yves Chibon e8bd26
                session=pagure.SESSION,
Pierre-Yves Chibon e8bd26
                username=user,
Pierre-Yves Chibon e8bd26
                fullname=user,
Pierre-Yves Chibon e8bd26
                default_email='%s@fedoraproject.org' % user,
Patrick Uiterwijk 0e5f4a
                keydir=pagure.APP.config.get('GITOLITE_KEYDIR', None),
Pierre-Yves Chibon e8bd26
            )
Pierre-Yves Chibon e8bd26
            pagure.SESSION.commit()
Pierre-Yves Chibon fa97f7
        except SQLAlchemyError as err:
Pierre-Yves Chibon e8bd26
            pagure.SESSION.rollback()
Pierre-Yves Chibon e8bd26
            print 'ERROR with user %s' % user
Pierre-Yves Chibon e8bd26
            print err
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
    for project in sorted(os.listdir(folder)):
Pierre-Yves Chibon e8bd26
        if debug:
Pierre-Yves Chibon e8bd26
            print project
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
        if not project.endswith('.git'):
Pierre-Yves Chibon e8bd26
            if debug:
Pierre-Yves Chibon f6632a
                print '  -skip: not a git repository'
Pierre-Yves Chibon e8bd26
            continue
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
        if project.split('.git')[0] not in pocs:
Pierre-Yves Chibon e8bd26
            if debug:
Pierre-Yves Chibon e8bd26
                print '  -skip: no pocs'
Pierre-Yves Chibon e8bd26
            continue
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
        try:
Pierre-Yves Chibon 8248d1
            name = project.split('.git')[0]
Pierre-Yves Chibon 0a6f4e
            orig_name = name
Pierre-Yves Chibon 0a6f4e
            name = 'rpms/%s' % name
Pierre-Yves Chibon 0a6f4e
            if name in pagure.APP.config['BLACKLISTED_PROJECTS']:
Pierre-Yves Chibon 0a6f4e
                raise pagure.exceptions.RepoExistsException(
Pierre-Yves Chibon 0a6f4e
                    'No project "%s" are allowed to be created due to potential '
Pierre-Yves Chibon 0a6f4e
                    'conflicts in URLs with pagure itself' % name
Pierre-Yves Chibon 0a6f4e
                )
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            user_obj = pagure.lib.__get_user(pagure.SESSION, pocs[orig_name])
Pierre-Yves Chibon 0a6f4e
            allowed_prefix = pagure.APP.config[
Pierre-Yves Chibon 0a6f4e
                'ALLOWED_PREFIX'] + [grp for grp in user_obj.groups]
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            first_part, _, second_part = name.partition('/')
Pierre-Yves Chibon 3c85bb
            if second_part and first_part not in allowed_prefix:
Pierre-Yves Chibon 0a6f4e
                raise pagure.exceptions.PagureException(
Jason Tibbitts 746662
                    'The prefix of your project must be in the list of allowed '
Jason Tibbitts 746662
                    'prefixes set by the admins of this pagure instance, or the name '
Jason Tibbitts 746662
                    'of a group of which you are a member.'
Pierre-Yves Chibon 0a6f4e
                )
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            gitfolder = pagure.APP.config['GIT_FOLDER']
Pierre-Yves Chibon 0a6f4e
            docfolder = pagure.APP.config['DOCS_FOLDER']
Pierre-Yves Chibon 0a6f4e
            ticketfolder = pagure.APP.config['TICKETS_FOLDER']
Pierre-Yves Chibon 0a6f4e
            requestfolder = pagure.APP.config['REQUESTS_FOLDER']
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            gitrepo = os.path.join(gitfolder, '%s.git' % name)
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            project = pagure.lib.model.Project(
Pierre-Yves Chibon 8248d1
                name=name,
Pierre-Yves Chibon 0a6f4e
                description=None,
Pierre-Yves Chibon 0a6f4e
                url=None,
Pierre-Yves Chibon 0a6f4e
                avatar_email=None,
Pierre-Yves Chibon 0a6f4e
                user_id=user_obj.id,
Pierre-Yves Chibon 0a6f4e
                parent_id=None,
Pierre-Yves Chibon 0a6f4e
                hook_token=pagure.lib.login.id_generator(40)
Pierre-Yves Chibon e8bd26
            )
Pierre-Yves Chibon 0a6f4e
            pagure.SESSION.add(project)
Pierre-Yves Chibon 0a6f4e
            # Make sure we won't have SQLAlchemy error before we create the repo
Pierre-Yves Chibon 0a6f4e
            pagure.SESSION.flush()
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok')
Pierre-Yves Chibon 0a6f4e
            if not os.path.exists(http_clone_file):
Pierre-Yves Chibon 0a6f4e
                with open(http_clone_file, 'w') as stream:
Pierre-Yves Chibon 0a6f4e
                    pass
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            docrepo = os.path.join(docfolder, project.path)
Pierre-Yves Chibon 0a6f4e
            if os.path.exists(docrepo):
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(gitrepo)
Pierre-Yves Chibon 0a6f4e
                raise pagure.exceptions.RepoExistsException(
Pierre-Yves Chibon 0a6f4e
                    'The docs repo "%s" already exists' % project.path
Pierre-Yves Chibon 0a6f4e
                )
Pierre-Yves Chibon 0a6f4e
            pygit2.init_repository(docrepo, bare=True)
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            ticketrepo = os.path.join(ticketfolder, project.path)
Pierre-Yves Chibon 0a6f4e
            if os.path.exists(ticketrepo):
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(gitrepo)
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(docrepo)
Pierre-Yves Chibon 0a6f4e
                raise pagure.exceptions.RepoExistsException(
Pierre-Yves Chibon 0a6f4e
                    'The tickets repo "%s" already exists' % project.path
Pierre-Yves Chibon 0a6f4e
                )
Pierre-Yves Chibon 0a6f4e
            pygit2.init_repository(
Pierre-Yves Chibon 0a6f4e
                ticketrepo, bare=True,
Pierre-Yves Chibon 0a6f4e
                mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP)
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon 0a6f4e
            requestrepo = os.path.join(requestfolder, project.path)
Pierre-Yves Chibon 0a6f4e
            if os.path.exists(requestrepo):
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(gitrepo)
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(docrepo)
Pierre-Yves Chibon 0a6f4e
                shutil.rmtree(ticketrepo)
Pierre-Yves Chibon 0a6f4e
                raise pagure.exceptions.RepoExistsException(
Pierre-Yves Chibon 0a6f4e
                    'The requests repo "%s" already exists' % project.path
Pierre-Yves Chibon 0a6f4e
                )
Pierre-Yves Chibon 0a6f4e
            pygit2.init_repository(
Pierre-Yves Chibon 0a6f4e
                requestrepo, bare=True,
Pierre-Yves Chibon 0a6f4e
                mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP)
Pierre-Yves Chibon 0a6f4e
Pierre-Yves Chibon e8bd26
            pagure.SESSION.commit()
Pierre-Yves Chibon fa97f7
        except pagure.exceptions.PagureException as err:
Pierre-Yves Chibon e8bd26
            print 'ERROR with project %s' % project
Pierre-Yves Chibon e8bd26
            print err
Pierre-Yves Chibon fa97f7
        except SQLAlchemyError as err:  # pragma: no cover
Pierre-Yves Chibon e8bd26
            pagure.SESSION.rollback()
Pierre-Yves Chibon e8bd26
            print 'ERROR (DB) with project %s' % project
Pierre-Yves Chibon e8bd26
            print err
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
if __name__ == '__main__':
Pierre-Yves Chibon e8bd26
    parser = argparse.ArgumentParser(
Pierre-Yves Chibon e8bd26
        description='Script creating projects on pagure based on the git '
Pierre-Yves Chibon e8bd26
        'repos present in the specified folder and the pkgdb information.'
Pierre-Yves Chibon e8bd26
    )
Pierre-Yves Chibon e8bd26
    parser.add_argument(
Pierre-Yves Chibon e8bd26
        'folder',
Pierre-Yves Chibon e8bd26
        help='Folder containing all the git repos of the projects to create')
Pierre-Yves Chibon e8bd26
    parser.add_argument(
Pierre-Yves Chibon e8bd26
        '--debug', dest='debug', action='store_true', default=False,
Pierre-Yves Chibon e8bd26
        help='Print the debugging output')
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
    args = parser.parse_args()
Pierre-Yves Chibon e8bd26
Pierre-Yves Chibon e8bd26
    main(args.folder, debug=args.debug)