diff --git a/pagure/hooks/__init__.py b/pagure/hooks/__init__.py index cc6fc29..d300b78 100644 --- a/pagure/hooks/__init__.py +++ b/pagure/hooks/__init__.py @@ -10,6 +10,8 @@ from __future__ import unicode_literals +import subprocess +import sys import os import six @@ -17,7 +19,9 @@ import wtforms from pagure.config import config as pagure_config from pagure.exceptions import FileNotFoundException -from pagure.utils import get_repo_path +import pagure.lib +import pagure.lib.git +from pagure.lib.plugins import get_enabled_plugins class RequiredIf(wtforms.validators.Required): @@ -51,43 +55,113 @@ class RequiredIf(wtforms.validators.Required): raise wtforms.validators.StopValidation(message) +class BaseRunner(object): + dbobj = None + + @classmethod + def runhook( + cls, session, username, hooktype, project, repotype, repodir, changes + ): + """ Run a specific hook on a project. + + By default, this calls out to the pre_receive, update or post_receive + functions as appropriate. + + Args: + session (Session): Database session + username (string): The user performing a push + project (model.Project): The project this call is made for + repotype (string): Value of lib.REPOTYPES indicating for which + repo the currnet call is + repodir (string): Directory where a clone of the specified repo is + located. Do note that this might or might not be a writable + clone. + changes (dict): A dict with keys being the ref to update, values + being a tuple of (from, to). + """ + if hooktype == "pre-receive": + cls.pre_receive( + session, username, project, repotype, repodir, changes + ) + elif hooktype == "update": + cls.update(session, username, project, repotype, repodir, changes) + elif hooktype == "post-receive": + cls.post_receive( + session, username, project, repotype, repodir, changes + ) + else: + raise ValueError('Invalid hook type "%s"' % hooktype) + + @staticmethod + def pre_receive(session, username, project, repotype, repodir, changes): + """ Run the pre-receive tasks of a hook. + + For args, see BaseRunner.runhook. + """ + pass + + @staticmethod + def update(session, username, project, repotype, repodir, changes): + """ Run the update tasks of a hook. + + For args, see BaseRunner.runhook. + Note that the "changes" list has exactly one element. + """ + pass + + @staticmethod + def post_receive(session, username, project, repotype, repodir, changes): + """ Run the post-receive tasks of a hook. + + For args, see BaseRunner.runhook. + """ + pass + + class BaseHook(object): """ Base class for pagure's hooks. """ name = None form = None description = None + db_object = None + # hook_type is not used in hooks that use a Runner class, as those can + # implement run actions on whatever is useful to them. hook_type = "post-receive" + runner = None @classmethod def set_up(cls, project): """ Install the generic post-receive hook that allow us to call multiple post-receive hooks as set per plugin. """ - repopaths = [get_repo_path(project)] - for folder in [ - pagure_config.get("DOCS_FOLDER"), - pagure_config.get("REQUESTS_FOLDER"), - ]: - if folder: - repopaths.append(os.path.join(folder, project.path)) + if project.is_on_repospanner: + # If the project is on repoSpanner, there's nothing to set up, + # as the hook script will be arranged by repo creation. + return + + repopaths = [ + project.repopath(repotype) for repotype in pagure.lib.REPOTYPES + ] hook_files = os.path.join( os.path.dirname(os.path.realpath(__file__)), "files" ) for repopath in repopaths: + if repopath is None: + continue + # Make sure the hooks folder exists hookfolder = os.path.join(repopath, "hooks") if not os.path.exists(hookfolder): os.makedirs(hookfolder) - # Install the main post-receive file - postreceive = os.path.join(hookfolder, cls.hook_type) - if not os.path.exists(postreceive): - os.symlink( - os.path.join(hook_files, cls.hook_type), postreceive - ) + for hooktype in ("pre-receive", "update", "post-receive"): + # Install the main hook file + target = os.path.join(hookfolder, hooktype) + if not os.path.exists(target): + os.symlink(os.path.join(hook_files, "hookrunner"), target) @classmethod def base_install(cls, repopaths, dbobj, hook_name, filein): @@ -99,6 +173,11 @@ class BaseHook(object): information. """ + if cls.runner: + # In the case of a new-style hook (with a Runner), there is no + # need to copy any files into place + return + for repopath in repopaths: if not os.path.exists(repopath): raise FileNotFoundException("Repo %s not found" % repopath) @@ -137,3 +216,173 @@ class BaseHook(object): ) if os.path.exists(hook_path): os.unlink(hook_path) + + @classmethod + def install(cls, *args): + """ In sub-classess, this can be used for installation of the hook. + + However, this is not required anymore for hooks with a Runner. + This class is here as backwards compatibility. + + All args are ignored. + """ + if not cls.runner: + raise ValueError("BaseHook.install called for runner-less hook") + + @classmethod + def remove(cls, *args): + """ In sub-classess, this can be used for removal of the hook. + + However, this is not required anymore for hooks with a Runner. + This class is here as backwards compatibility. + + All args are ignored. + """ + if not cls.runner: + raise ValueError("BaseHook.remove called for runner-less hook") + + +def run_project_hooks( + session, username, project, hooktype, repotype, repodir, changes +): + """ Function to run the hooks on a project + + This will first call all the plugins with a Runner on the project, + and afterwards, for a non-repoSpanner repo, run all hooks/.* + scripts in the repo. + + Args: + session: Database session + username (string): The user performing a push + project (model.Project): The project this call is made for + repotype (string): Value of lib.REPOTYPES indicating for which + repo the currnet call is + repodir (string): Directory where a clone of the specified repo is + located. Do note that this might or might not be a writable + clone. + hooktype (string): The type of hook to run: pre-receive, update + or post-receive + changes (dict): A dict with keys being the ref to update, values being + a tuple of (from, to). + """ + debug = pagure_config.get("HOOK_DEBUG", False) + + for plugin, _ in get_enabled_plugins(project): + if not plugin.runner: + if debug: + print( + "Hook plugin %s should be ported to Runner" % plugin.name + ) + else: + if debug: + print("Running plugin %s" % plugin.name) + + plugin.runner.runhook( + hooktype, project, repotype, repodir, changes + ) + + if project.is_on_repospanner: + # We are done. We are not doing any legacy hooks for repoSpanner + return + + hookdir = os.path.join(repodir, "hooks") + if not os.path.exists(hookdir): + return + + stdin = "" + args = [] + if hooktype == "update": + refname = six.next(six.iterkeys(changes)) + (revfrom, revto) = changes[refname] + args = [refname, revfrom, revto] + else: + stdin = ( + "\n".join( + [ + "%s %s %s" % (changes[refname] + (refname,)) + for refname in changes + ] + ) + + "\n" + ) + stdin = stdin.encode("utf-8") + + if debug: + print("Running legacy hooks with args: %s, stdin: %s" % (args, stdin)) + + for hook in os.listdir(hookdir): + # This is for legacy hooks, which create symlinks in the form of + # "post-receive.$pluginname" + if hook.startswith(hooktype + "."): + hookfile = os.path.join(hookdir, hook) + + # Determine if this is an actual hook, or if it's a remnant + # from a hook that was installed before it was moved to the + # runner system. + if os.path.realpath(hookfile) == pagure.lib.HOOK_DNE_TARGET: + continue + + # Execute + print( + "Running legacy hook %s. " + "Please ask your admin to port this to the new plugin " + "format, as the current system will cease functioning " + "in a future Pagure release" % hook + ) + + # Using subprocess.Popen rather than check_call so that stdin + # can be passed without having to use a temporary file. + proc = subprocess.Popen( + [hookfile] + args, cwd=repodir, stdin=subprocess.PIPE + ) + proc.communicate(stdin) + ecode = proc.wait() + # The legacy script would error out the entire hook running if + # one failed, so let's keep doing the same for backwards compat + if ecode != 0: + print("Hook %s errored out" % hook) + raise SystemExit(1) + + +def run_hook_file(hooktype): + """ Runs a specific hook by grabbing the changes and running functions. + + Args: + hooktype (string): The name of the hook to run: pre-receive, update + or post-receive + """ + changes = {} + if hooktype in ("pre-receive", "post-receive"): + for line in sys.stdin: + (oldrev, newrev, refname) = line.strip().split(" ", 2) + changes[refname] = (oldrev, newrev) + elif hooktype == "update": + (refname, oldrev, newrev) = sys.argv[1:] + changes[refname] = (oldrev, newrev) + else: + raise ValueError("Hook type %s not valid" % hooktype) + + pushuser = os.environ.get("GL_USER") + + if pagure_config.get("HOOK_DEBUG", False): + print("Changes: %s" % changes) + + gitdir = os.path.abspath(os.environ["GIT_DIR"]) + ( + repotype, + username, + namespace, + repo, + ) = pagure.lib.git.get_repo_info_from_path(gitdir) + + session = pagure.lib.create_session(pagure_config["DB_URL"]) + if not session: + raise Exception("Unable to initialize db session") + + project = pagure.lib._get_project( + session, repo, user=username, namespace=namespace + ) + + run_project_hooks( + session, pushuser, project, hooktype, repotype, gitdir, changes + ) diff --git a/pagure/hooks/files/hookrunner b/pagure/hooks/files/hookrunner new file mode 100755 index 0000000..2a2b7ce --- /dev/null +++ b/pagure/hooks/files/hookrunner @@ -0,0 +1,19 @@ +#!/bin/env python3 +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + +""" +import os +import sys + +import pagure.lib +from pagure.hooks import run_hook_file + +hooktype = os.path.basename(sys.argv[0]) + +run_hook_file(hooktype) diff --git a/pagure/hooks/files/post-receive b/pagure/hooks/files/post-receive deleted file mode 100755 index 8a63e5c..0000000 --- a/pagure/hooks/files/post-receive +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# author: orefalo - -hookname=`basename $0` - - -FILE=`mktemp` -trap 'rm -f $FILE' EXIT -cat - > $FILE - -for hook in $GIT_DIR/hooks/$hookname.* -do - if test -x "$hook"; then - cat $FILE | $hook "$@" - status=$? - - if test $status -ne 0; then - echo Hook $hook failed with error code $status - exit $status - fi - fi -done diff --git a/pagure/hooks/files/post-receive b/pagure/hooks/files/post-receive new file mode 120000 index 0000000..936f169 --- /dev/null +++ b/pagure/hooks/files/post-receive @@ -0,0 +1 @@ +hookrunner \ No newline at end of file diff --git a/pagure/hooks/files/pre-receive b/pagure/hooks/files/pre-receive deleted file mode 100755 index 8a63e5c..0000000 --- a/pagure/hooks/files/pre-receive +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# author: orefalo - -hookname=`basename $0` - - -FILE=`mktemp` -trap 'rm -f $FILE' EXIT -cat - > $FILE - -for hook in $GIT_DIR/hooks/$hookname.* -do - if test -x "$hook"; then - cat $FILE | $hook "$@" - status=$? - - if test $status -ne 0; then - echo Hook $hook failed with error code $status - exit $status - fi - fi -done diff --git a/pagure/hooks/files/pre-receive b/pagure/hooks/files/pre-receive new file mode 120000 index 0000000..936f169 --- /dev/null +++ b/pagure/hooks/files/pre-receive @@ -0,0 +1 @@ +hookrunner \ No newline at end of file diff --git a/pagure/hooks/files/rtd_hook.py b/pagure/hooks/files/rtd_hook.py deleted file mode 100755 index 00b0b03..0000000 --- a/pagure/hooks/files/rtd_hook.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - - -"""Pagure specific hook to trigger a build on a readthedocs.org project. -""" - -from __future__ import print_function, unicode_literals - -import os -import sys - -import requests - - -if "PAGURE_CONFIG" not in os.environ and os.path.exists( - "/etc/pagure/pagure.cfg" -): - os.environ["PAGURE_CONFIG"] = "/etc/pagure/pagure.cfg" - - -import pagure # noqa: E402 -import pagure.exceptions # noqa: E402 -import pagure.lib.link # noqa: E402 -import pagure.lib.plugins # noqa: E402 - -_config = pagure.config.config -abspath = os.path.abspath(os.environ["GIT_DIR"]) - - -def run_as_post_receive_hook(): - reponame = pagure.lib.git.get_repo_name(abspath) - username = pagure.lib.git.get_username(abspath) - namespace = pagure.lib.git.get_repo_namespace(abspath) - session = pagure.lib.create_session(_config["DB_URL"]) - if _config.get("HOOK_DEBUG", False): - print("repo: ", reponame) - print("user: ", username) - print("namespace:", namespace) - - repo = pagure.lib.get_authorized_project( - session, reponame, user=username, namespace=namespace - ) - if not repo: - print("Unknown repo %s of username: %s" % (reponame, username)) - session.close() - sys.exit(1) - - hook = pagure.lib.plugins.get_plugin("Read the Doc") - hook.db_object() - - # Get the list of branches - branches = [ - branch.strip() - for branch in repo.rtd_hook.branches.split(",") - if repo.rtd_hook - ] - - # Remove empty branches - branches = [branch.strip() for branch in branches if branch] - - url = repo.rtd_hook.api_url - if not url: - print( - "No API url specified to trigger the build, please update " - "the configuration" - ) - session.close() - return 1 - if not repo.rtd_hook.api_token: - print( - "No API token specified to trigger the build, please update " - "the configuration" - ) - session.close() - return 1 - - for line in sys.stdin: - if _config.get("HOOK_DEBUG", False): - print(line) - (oldrev, newrev, refname) = line.strip().split(" ", 2) - - refname = refname.replace("refs/heads/", "") - if branches: - if refname in branches: - print("Starting RTD build at %s" % (url)) - requests.post( - url, - data={ - "branches": refname, - "token": repo.rtd_hook.api_token, - }, - timeout=60, - ) - else: - print("Starting RTD build at %s" % (url)) - requests.post( - url, - data={"branches": refname, "token": repo.rtd_hook.api_token}, - timeout=60, - ) - - session.close() - - -def main(args): - run_as_post_receive_hook() - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/pagure/hooks/files/rtd_hook.py b/pagure/hooks/files/rtd_hook.py new file mode 120000 index 0000000..6be2006 --- /dev/null +++ b/pagure/hooks/files/rtd_hook.py @@ -0,0 +1 @@ +/does/not/exist \ No newline at end of file diff --git a/pagure/hooks/rtd.py b/pagure/hooks/rtd.py index 37a333c..85d034c 100644 --- a/pagure/hooks/rtd.py +++ b/pagure/hooks/rtd.py @@ -12,6 +12,7 @@ from __future__ import unicode_literals import sqlalchemy as sa +import requests import wtforms try: @@ -21,9 +22,11 @@ except ImportError: from sqlalchemy.orm import relation from sqlalchemy.orm import backref -from pagure.hooks import BaseHook +import pagure +from pagure.hooks import BaseHook, BaseRunner from pagure.lib.model import BASE, Project -from pagure.utils import get_repo_path + +_config = pagure.config.config class RtdTable(BASE): @@ -87,7 +90,7 @@ If you specify one or more branches (using commas `,` to separate them) only pushes made to these branches will trigger a new build of the documentation. To set up this hook, you will need to login to https://readthedocs.org/ -Go to your project's adming settings, and in the ``Integrations`` section +Go to your project's admin settings, and in the ``Integrations`` section add a new ``Generic API incoming webhook``. This will give you access to one URL and one API token, both of which you @@ -96,6 +99,62 @@ will have to provide below. """ +class RtdRunner(BaseRunner): + @staticmethod + def post_receive(session, username, project, repotype, repodir, changes): + """ Perform the RTD Post Receive hook. + + For arguments, see BaseRunner.runhook. + """ + # Get the list of branches + branches = [ + branch.strip() for branch in project.rtd_hook.branches.split(",") + ] + + # Remove empty branches + branches = [branch.strip() for branch in branches if branch] + + url = project.rtd_hook.api_url + if not url: + print( + "No API url specified to trigger the build, please update " + "the configuration" + ) + if not project.rtd_hook.api_token: + print( + "No API token specified to trigger the build, please update " + "the configuration" + ) + + for refname in changes: + oldrev, newrev = changes[refname] + if _config.get("HOOK_DEBUG", False): + print("%s: %s -> %s" % (refname, oldrev, newrev)) + + refname = refname.replace("refs/heads/", "") + if branches: + if refname in branches: + print("Starting RTD build at %s" % (url)) + requests.post( + url, + data={ + "branches": refname, + "token": project.rtd_hook.api_token, + }, + timeout=60, + ) + else: + print("Starting RTD build at %s" % (url)) + requests.post( + url, + data={ + "branches": refname, + "token": project.rtd_hook.api_token, + }, + timeout=60, + ) + + class RtdHook(BaseHook): """ Read The Doc hook. """ @@ -103,29 +162,6 @@ class RtdHook(BaseHook): description = DESCRIPTION form = RtdForm db_object = RtdTable + runner = RtdRunner backref = "rtd_hook" form_fields = ["active", "api_url", "api_token", "branches"] - - @classmethod - def install(cls, project, dbobj): - """ Method called to install the hook for a project. - - :arg project: a ``pagure.model.Project`` object to which the hook - should be installed - - """ - repopaths = [get_repo_path(project)] - - cls.base_install(repopaths, dbobj, "rtd", "rtd_hook.py") - - @classmethod - def remove(cls, project): - """ Method called to remove the hook of a project. - - :arg project: a ``pagure.model.Project`` object to which the hook - should be installed - - """ - repopaths = [get_repo_path(project)] - - cls.base_remove(repopaths, "rtd") diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index b0b5189..8438423 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -69,6 +69,9 @@ REDIS = None PAGURE_CI = None REPOTYPES = ("main", "docs", "tickets", "requests") _log = logging.getLogger(__name__) +# The target for hooks migrated to the Runner system, to be able to detect +# whether a hook was migrated without having to open and read the file +HOOK_DNE_TARGET = "/does/not/exist" class Unspecified(object): diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 5616b7b..bba1a38 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -29,6 +29,8 @@ import pygit2 import six from sqlalchemy.exc import SQLAlchemyError + +# from sqlalchemy.orm.session import Session from pygit2.remote import RemoteCollection import pagure.utils @@ -40,6 +42,8 @@ from pagure.lib import model from pagure.lib.repo import PagureRepo from pagure.lib import tasks +# from pagure.hooks import run_project_hooks + _log = logging.getLogger(__name__) @@ -972,7 +976,13 @@ class TemporaryClone(object): if self._project.is_on_repospanner: return - return # TODO: Enable + # TODO: Rework this to make use of run_project_hooks + # session = Session.object_session(self._project) + # run_project_hooks( + # session, + # self._project, + # 'pre-receive', + # ) gitrepo_obj = PagureRepo(self._origpath) gitrepo_obj.run_hook(parent, commit, "refs/heads/%s" % branch, user) @@ -1229,46 +1239,124 @@ def get_commit_subject(commit, abspath): return subject -def get_repo_name(abspath): - """ Return the name of the git repo based on its path. - """ - repo_name = ".".join( - abspath.rsplit(os.path.sep, 1)[-1].rsplit(".", 1)[:-1] - ) - return repo_name +def get_repo_info_from_path(gitdir): + """ Returns the name, username, namespace and type of a git directory + This gets computed based on the *_FOLDER's in the config file, + and as such only works for the central file-based repositories. -def get_repo_namespace(abspath, gitfolder=None): - """ Return the name of the git repo based on its path. + Args: + gitdir (string): Path of the canonical git repository + Return: (tuple): Tuple with (repotype, username, namespace, repo) + Some of these elements may be None if not applicable. """ + if not os.path.isabs(gitdir): + raise ValueError("Tried to locate non-absolute gitdir %s" % gitdir) + gitdir = os.path.normpath(gitdir) + + types = { + "main": os.path.abspath(pagure_config["GIT_FOLDER"]), + "docs": os.path.abspath(pagure_config["DOCS_FOLDER"]), + "tickets": os.path.abspath(pagure_config["TICKETS_FOLDER"]), + "requests": os.path.abspath(pagure_config["REQUESTS_FOLDER"]), + } + + match = None + matchlen = None + + # First find the longest match in types. This makes sure that even if the + # non-main repos are in a subdir of main (i.e. repos/ and repos/tickets/), + # we find the correct type. + for typename in types: + path = types[typename] + "/" + if gitdir.startswith(path) and ( + matchlen is None or len(path) > matchlen + ): + match = typename + matchlen = len(path) + + if match is None: + raise ValueError("Gitdir %s could not be located") + + typepath = types[match] + guesspath = gitdir[len(typepath) + 1 :] + if len(guesspath) < 5: + # At least 4 characters for ".git" is required plus one for project + # name + raise ValueError("Invalid gitdir %s located" % gitdir) + + # Just in the case we run on a non-*nix system... + guesspath = guesspath.replace("\\", "/") + + # Now guesspath should be one of: + # - reponame.git + # - namespace/reponame.git + # - forks/username/reponame.git + # - forks/username/namespace/reponame.git + repotype = match + username = None namespace = None - if not gitfolder: - gitfolder = pagure_config["GIT_FOLDER"] + repo = None - short_path = ( - os.path.realpath(abspath) - .replace(os.path.realpath(gitfolder), "") - .strip("/") + guesspath, repo = os.path.split(guesspath) + if not repo.endswith(".git"): + raise ValueError("Git dir looks to not be a bare repo") + repo = repo[: -len(".git")] + if not repo: + raise ValueError("Gitdir %s seems to not be a bare repo" % gitdir) + + # Split the guesspath up, throwing out any empty strings + splitguess = [part for part in guesspath.split("/") if part] + if splitguess and splitguess[0] == "forks": + if len(splitguess) < 2: + raise ValueError("Invalid gitdir %s" % gitdir) + username = splitguess[1] + splitguess = splitguess[2:] + + if splitguess: + # At this point, we've cut off the repo name at the end, and any forks/ + # indicators and their usernames are also removed, so remains just the + # namespace + namespace = os.path.join(*splitguess) + + # Okay, we think we have everything. Let's make doubly sure the path is + # correct and exists + rebuiltpath = os.path.join( + typepath, + "forks/" if username else "", + username if username else "", + namespace if namespace else "", + repo + ".git", ) + if os.path.normpath(rebuiltpath) != gitdir: + raise ValueError( + "Rebuilt %s path not identical to gitdir %s" + % (rebuiltpath, gitdir) + ) + if not os.path.exists(rebuiltpath): + raise ValueError("Splitting gitdir %s failed" % gitdir) - if short_path.startswith("forks/"): - username, projectname = short_path.split("forks/", 1)[1].split("/", 1) - else: - projectname = short_path + return (repotype, username, namespace, repo) + + +def get_repo_name(abspath): + """ Return the name of the git repo based on its path. + """ + _, _, _, name = get_repo_info_from_path(abspath) + return name - if "/" in projectname: - namespace = projectname.rsplit("/", 1)[0] +def get_repo_namespace(abspath, gitfolder=None): + """ Return the name of the git repo based on its path. + """ + _, _, namespace, _ = get_repo_info_from_path(abspath) return namespace def get_username(abspath): """ Return the username of the git repo based on its path. """ - username = None - repo = os.path.abspath(os.path.join(abspath, "..")) - if "/forks/" in repo: - username = repo.split("/forks/", 1)[1].split("/", 1)[0] + _, username, _, _ = get_repo_info_from_path(abspath) return username diff --git a/pagure/lib/model.py b/pagure/lib/model.py index 984b5b9..d23f426 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -26,9 +26,8 @@ import os import six import sqlalchemy as sa -from sqlalchemy import create_engine, MetaData +from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import backref from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import scoped_session @@ -36,20 +35,11 @@ from sqlalchemy.orm import relation import pagure.exceptions from pagure.config import config as pagure_config +from pagure.lib.model_base import BASE +from pagure.lib.plugins import get_plugin_tables from pagure.utils import is_true -CONVENTION = { - "ix": "ix_%(table_name)s_%(column_0_label)s", - # Checks are currently buggy and prevent us from naming them correctly - # "ck": "ck_%(table_name)s_%(constraint_name)s", - "fk": "%(table_name)s_%(column_0_name)s_fkey", - "pk": "%(table_name)s_pkey", - "uq": "%(table_name)s_%(column_0_name)s_key", -} - -BASE = declarative_base(metadata=MetaData(naming_convention=CONVENTION)) - _log = logging.getLogger(__name__) # hit w/ all the id field we use @@ -79,8 +69,6 @@ def create_tables(db_url, alembic_ini=None, acls=None, debug=False): else: # pragma: no cover engine = create_engine(db_url, echo=debug) - from pagure.lib.plugins import get_plugin_tables - get_plugin_tables() BASE.metadata.create_all(engine) # engine.execute(collection_package_create_view(driver=engine.driver)) @@ -2991,3 +2979,7 @@ class PagureUserGroup(BASE): # Constraints __table_args__ = (sa.UniqueConstraint("user_id", "group_id"),) + + +# Make sure to load the Plugin tables, so they have a chance to register +get_plugin_tables() diff --git a/pagure/lib/model_base.py b/pagure/lib/model_base.py new file mode 100644 index 0000000..1bc0a04 --- /dev/null +++ b/pagure/lib/model_base.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2014-2018 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals + +from sqlalchemy import MetaData +from sqlalchemy.ext.declarative import declarative_base + +CONVENTION = { + "ix": "ix_%(table_name)s_%(column_0_label)s", + # Checks are currently buggy and prevent us from naming them correctly + # "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "%(table_name)s_%(column_0_name)s_fkey", + "pk": "%(table_name)s_pkey", + "uq": "%(table_name)s_%(column_0_name)s_key", +} + +BASE = declarative_base(metadata=MetaData(naming_convention=CONVENTION)) diff --git a/pagure/lib/plugins.py b/pagure/lib/plugins.py index e154c2a..394f449 100644 --- a/pagure/lib/plugins.py +++ b/pagure/lib/plugins.py @@ -12,7 +12,7 @@ from __future__ import unicode_literals from straight.plugin import load -from pagure.lib.model import BASE +from pagure.lib.model_base import BASE def get_plugin_names(blacklist=None): @@ -48,3 +48,22 @@ def get_plugin(plugin_name): for plugin in plugins: if plugin.name == plugin_name: return plugin + + +def get_enabled_plugins(project): + """ Returns a list of plugins enabled for a specific project. + + Args: + project (model.Project): The project to look for. + Returns: (list): A list of tuples (pluginclass, dbobj) with the plugin + classess and dbobjects for plugins enabled for the project. + """ + from pagure.hooks import BaseHook + + enabled = [] + for plugin in load("pagure.hooks", subclasses=BaseHook): + if plugin.db_object and hasattr(project, plugin.backref): + dbobj = getattr(project, plugin.backref) + if dbobj: + enabled.append((plugin, dbobj)) + return enabled diff --git a/tests/__init__.py b/tests/__init__.py index cadb5df..f0e9553 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -60,6 +60,7 @@ from pagure.api.ci import jenkins import pagure.flask_app import pagure.lib import pagure.lib.model +import pagure.lib.tasks_mirror import pagure.perfrepo as perfrepo from pagure.config import config as pagure_config, reload_config from pagure.lib.repo import PagureRepo diff --git a/tests/test_pagure_flask_ui_plugins_rtd_hook.py b/tests/test_pagure_flask_ui_plugins_rtd_hook.py index ce61a18..068ec5c 100644 --- a/tests/test_pagure_flask_ui_plugins_rtd_hook.py +++ b/tests/test_pagure_flask_ui_plugins_rtd_hook.py @@ -120,10 +120,6 @@ class PagureFlaskPluginRtdHooktests(tests.SimplePagureTest): '', output_text) - self.assertTrue(os.path.exists(os.path.join( - self.path, 'repos', 'test.git', 'hooks', - 'post-receive.rtd'))) - # De-Activate hook data = {'csrf_token': csrf_token} output = self.app.post( diff --git a/tests/test_pagure_lib_git.py b/tests/test_pagure_lib_git.py index bf3396d..2804441 100644 --- a/tests/test_pagure_lib_git.py +++ b/tests/test_pagure_lib_git.py @@ -2689,69 +2689,47 @@ index 0000000..60f7480 def test_get_repo_name(self): """ Test the get_repo_name method of pagure.lib.git. """ - gitrepo = os.path.join(self.path, 'tickets', 'test_ticket_repo.git') - repo_name = pagure.lib.git.get_repo_name(gitrepo) - self.assertEqual(repo_name, 'test_ticket_repo') - repo_name = pagure.lib.git.get_repo_name('foo/bar/baz/test.git') - self.assertEqual(repo_name, 'test') + def runtest(reponame, *path): + gitrepo = os.path.join(self.path, 'repos', *path) + os.makedirs(gitrepo) + repo_name = pagure.lib.git.get_repo_name(gitrepo) + self.assertEqual(repo_name, reponame) - repo_name = pagure.lib.git.get_repo_name('foo.test.git') - self.assertEqual(repo_name, 'foo.test') + runtest('test_ticket_repo', 'tickets', 'test_ticket_repo.git') + runtest('test', 'test.git') + runtest('foo.test', 'foo.test.git') def test_get_username(self): """ Test the get_username method of pagure.lib.git. """ - gitrepo = os.path.join(self.path, 'tickets', 'test_ticket_repo.git') - repo_name = pagure.lib.git.get_username(gitrepo) - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_username('foo/bar/baz/test.git') - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_username('foo.test.git') - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_username( - os.path.join(self.path, 'repos', 'forks', 'pingou', 'foo.test.git')) - self.assertEqual(repo_name, 'pingou') - - repo_name = pagure.lib.git.get_username( - os.path.join(self.path, 'repos', 'forks', 'pingou', 'bar/foo.test.git')) - self.assertEqual(repo_name, 'pingou') - - repo_name = pagure.lib.git.get_username(os.path.join( - self.path, 'repos', 'forks', 'pingou', 'fooo/bar/foo.test.git')) - self.assertEqual(repo_name, 'pingou') + def runtest(username, *path): + gitrepo = os.path.join(self.path, 'repos', *path) + os.makedirs(gitrepo) + repo_username = pagure.lib.git.get_username(gitrepo) + self.assertEqual(repo_username, username) + + runtest(None, 'tickets', 'test_ticket_repo.git') + runtest(None, 'test.git') + runtest(None, 'foo.test.git') + runtest('pingou', 'forks', 'pingou', 'foo.test.git') + runtest('pingou', 'forks', 'pingou', 'bar/foo.test.git') def test_get_repo_namespace(self): """ Test the get_repo_namespace method of pagure.lib.git. """ - repo_name = pagure.lib.git.get_repo_namespace( - os.path.join(self.path, 'repos', 'test_ticket_repo.git')) - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_repo_namespace( - os.path.join(self.path, 'repos', 'foo/bar/baz/test.git')) - self.assertEqual(repo_name, 'foo/bar/baz') - - repo_name = pagure.lib.git.get_repo_namespace( - os.path.join(self.path, 'repos', 'foo.test.git')) - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_repo_namespace(os.path.join( - self.path, 'repos', 'forks', 'user', 'foo.test.git')) - self.assertEqual(repo_name, None) - - repo_name = pagure.lib.git.get_repo_namespace(os.path.join( - self.path, 'repos', 'forks', 'user', 'bar/foo.test.git')) - self.assertEqual(repo_name, 'bar') - - repo_name = pagure.lib.git.get_repo_namespace(os.path.join( - self.path, 'repos', 'forks', 'user', 'ns/bar/foo.test.git')) - self.assertEqual(repo_name, 'ns/bar') - - repo_name = pagure.lib.git.get_repo_namespace(os.path.join( - self.path, 'repos', 'forks', 'user', '/bar/foo.test.git')) - self.assertEqual(repo_name, 'bar') + def runtest(namespace, *path): + gitrepo = os.path.join(self.path, 'repos', *path) + if not os.path.exists(gitrepo): + os.makedirs(gitrepo) + repo_namespace = pagure.lib.git.get_repo_namespace(gitrepo) + self.assertEqual(repo_namespace, namespace) + + runtest(None, 'test_ticket_repo.git') + runtest('foo/bar/baz', 'foo', 'bar', 'baz', 'test.git') + runtest(None, 'foo.test.git') + runtest(None, 'forks', 'user', 'foo.test.git') + runtest('bar', 'forks', 'user', 'bar', 'foo.test.git') + runtest('ns/bar', 'forks', 'user', 'ns', 'bar', 'foo.test.git') + runtest('bar', 'forks', 'user', 'bar', 'foo.test.git') def test_update_custom_fields_from_json(self): """ Test the update_custom_fields_from_json method of lib.git """