From 7b908095e395fae5b74fe99ef96839dc64d5a8db Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 31 2017 09:42:33 +0000 Subject: Add explicit project-level locking with lock types This offers different types of locking within a project. Specifically, it prevents the worker from locking up the frontends when it gets a project-wide lock, while still ensuring that multiple workers can't work on the same project. Signed-off-by: Patrick Uiterwijk --- diff --git a/alembic/versions/5179e99d35a5_add_lock_types.py b/alembic/versions/5179e99d35a5_add_lock_types.py new file mode 100644 index 0000000..ae991ec --- /dev/null +++ b/alembic/versions/5179e99d35a5_add_lock_types.py @@ -0,0 +1,45 @@ +"""Add lock types + +Revision ID: 5179e99d35a5 +Revises: d4d2c5aa8a0 +Create Date: 2017-05-30 14:47:55.063908 + +""" + +# revision identifiers, used by Alembic. +revision = '5179e99d35a5' +down_revision = 'd4d2c5aa8a0' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'project_locks', + sa.Column('project_id', + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE' + ), + nullable=False, + primary_key=True + ), + sa.Column('lock_type', + sa.Enum( + 'WORKER', + name='lock_type_enum' + ), + nullable=False, + primary_key=True + ) + ) + + # Add WORKER locks everywhere + conn = op.get_bind() + conn.execute("""INSERT INTO project_locks (project_id, lock_type) + SELECT id, 'WORKER' from projects""") + + +def downgrade(): + op.drop_table('project_locks') diff --git a/pagure/default_config.py b/pagure/default_config.py index 3e8f546..d0c59b0 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -273,6 +273,10 @@ EXCLUDE_GROUP_INDEX = [] TRIGGER_CI = ['pretty please pagure-ci rebuild'] +# Never enable this option, this is intended for tests only, and can allow +# easy denial of service to the system if enabled. +ALLOW_PROJECT_DOWAIT = False + LOGGING = { 'version': 1, diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 363a644..f074ccd 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1350,8 +1350,14 @@ def new_project(session, user, name, blacklist, allowed_prefix, hook_token=pagure.lib.login.id_generator(40) ) session.add(project) - # Make sure we won't have SQLAlchemy error before we create the repo + # Flush so that a project ID is generated session.flush() + for ltype in model.ProjectLock.lock_type.type.enums: + lock = model.ProjectLock( + project_id=project.id, + lock_type=ltype) + session.add(lock) + session.commit() # Register creation et al log_action(session, 'created', project, user_obj) @@ -1958,7 +1964,7 @@ def search_projects( return query.all() -def _get_project(session, name, user=None, namespace=None, with_lock=False): +def _get_project(session, name, user=None, namespace=None): '''Get a project from the database ''' query = session.query( @@ -1974,10 +1980,6 @@ def _get_project(session, name, user=None, namespace=None, with_lock=False): else: query = query.filter(model.Project.namespace == namespace) - if with_lock: - query = query.with_for_update(nowait=False, - read=False) - if user is not None: query = query.filter( model.User.user == user diff --git a/pagure/lib/model.py b/pagure/lib/model.py index ba2cef3..3485785 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -800,6 +800,11 @@ class Project(BASE): 'ticket': self.get_project_groups(access='ticket', combine=False), } + def lock(self, ltype): + """ Get a SQL lock of type ltype for the current project. + """ + return ProjectLocker(self, ltype) + def to_json(self, public=False, api=False): ''' Return a representation of the project as JSON. ''' @@ -831,6 +836,74 @@ class Project(BASE): return output +class ProjectLock(BASE): + """ Table used to define project-specific locks. + + Table -- project_locks + """ + __tablename__ = 'project_locks' + + project_id = sa.Column( + sa.Integer, + sa.ForeignKey( + 'projects.id', onupdate='CASCADE', ondelete='CASCADE' + ), + nullable=False, + primary_key=True) + lock_type = sa.Column( + sa.Enum( + 'WORKER', + name='lock_type_enum', + ), + nullable=False, + primary_key=True) + + +class ProjectLocker(object): + """ This is used as a context manager to lock a project. + + This is used as a context manager to make it very explicit when we unlock + the project, and so that we unlock even if an exception occurs. + """ + def __init__(self, project, ltype): + self.session = None + self.lock = None + self.project_id = project.id + self.ltype = ltype + + def __enter__(self): + from pagure.lib import create_session + + self.session = create_session() + + _log.info('Grabbing lock for %d', self.project_id) + query = self.session.query( + ProjectLock + ).filter( + ProjectLock.project_id == self.project_id + ).filter( + ProjectLock.lock_type == self.ltype + ).with_for_update(nowait=False, + read=False) + + try: + self.lock = query.one() + except: + pl = ProjectLock( + project_id=self.project_id, lock_type=self.ltype) + self.session.add(pl) + self.session.commit() + self.lock = query.one() + + assert self.lock is not None + _log.info('Got lock for %d: %s', self.project_id, self.lock) + + def __exit__(self, *exargs): + _log.info('Releasing lock for %d', self.project_id) + self.session.remove() + _log.info('Released lock for %d', self.project_id) + + class ProjectUser(BASE): """ Stores the user of a projects. diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 1dc02ec..07923f7 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -12,6 +12,7 @@ import gc import os import os.path import shutil +import time from celery import Celery from celery.result import AsyncResult @@ -66,87 +67,89 @@ def create_project(username, namespace, name, add_readme, session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, - name=name, with_lock=True) - userobj = pagure.lib.search_user(session, username=username) - gitrepo = os.path.join(APP.config['GIT_FOLDER'], project.path) - - # Add the readme file if it was asked - if not add_readme: - pygit2.init_repository(gitrepo, bare=True) - else: - temp_gitrepo_path = tempfile.mkdtemp(prefix='pagure-') - temp_gitrepo = pygit2.init_repository(temp_gitrepo_path, bare=False) - author = userobj.fullname or userobj.user - author_email = userobj.default_email - if six.PY2: - author = author.encode('utf-8') - author_email = author_email.encode('utf-8') - author = pygit2.Signature(author, author_email) - content = u"# %s\n\n%s" % (name, project.description) - readme_file = os.path.join(temp_gitrepo.workdir, "README.md") - with open(readme_file, 'wb') as stream: - stream.write(content.encode('utf-8')) - temp_gitrepo.index.add_all() - temp_gitrepo.index.write() - tree = temp_gitrepo.index.write_tree() - temp_gitrepo.create_commit( - 'HEAD', author, author, 'Added the README', tree, []) - pygit2.clone_repository(temp_gitrepo_path, gitrepo, bare=True) - shutil.rmtree(temp_gitrepo_path) - - # Make the repo exportable via apache - http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') - if not os.path.exists(http_clone_file): - with open(http_clone_file, 'w') as stream: - pass - - docrepo = os.path.join(APP.config['DOCS_FOLDER'], project.path) - if os.path.exists(docrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - raise pagure.exceptions.RepoExistsException( - 'The docs repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository(docrepo, bare=True) - - ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], project.path) - if os.path.exists(ticketrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - shutil.rmtree(docrepo) - raise pagure.exceptions.RepoExistsException( - 'The tickets repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository( - ticketrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], project.path) - if os.path.exists(requestrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - shutil.rmtree(docrepo) - shutil.rmtree(ticketrepo) - raise pagure.exceptions.RepoExistsException( - 'The requests repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository( - requestrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - # Install the default hook - plugin = pagure.lib.plugins.get_plugin('default') - dbobj = plugin.db_object() - dbobj.active = True - dbobj.project_id = project.id - session.add(dbobj) - session.flush() - plugin.set_up(project) - plugin.install(project, dbobj) - session.commit() + name=name) + with project.lock('WORKER'): + userobj = pagure.lib.search_user(session, username=username) + gitrepo = os.path.join(APP.config['GIT_FOLDER'], project.path) + + # Add the readme file if it was asked + if not add_readme: + pygit2.init_repository(gitrepo, bare=True) + else: + temp_gitrepo_path = tempfile.mkdtemp(prefix='pagure-') + temp_gitrepo = pygit2.init_repository(temp_gitrepo_path, + bare=False) + author = userobj.fullname or userobj.user + author_email = userobj.default_email + if six.PY2: + author = author.encode('utf-8') + author_email = author_email.encode('utf-8') + author = pygit2.Signature(author, author_email) + content = u"# %s\n\n%s" % (name, project.description) + readme_file = os.path.join(temp_gitrepo.workdir, "README.md") + with open(readme_file, 'wb') as stream: + stream.write(content.encode('utf-8')) + temp_gitrepo.index.add_all() + temp_gitrepo.index.write() + tree = temp_gitrepo.index.write_tree() + temp_gitrepo.create_commit( + 'HEAD', author, author, 'Added the README', tree, []) + pygit2.clone_repository(temp_gitrepo_path, gitrepo, bare=True) + shutil.rmtree(temp_gitrepo_path) + + # Make the repo exportable via apache + http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w') as stream: + pass + + docrepo = os.path.join(APP.config['DOCS_FOLDER'], project.path) + if os.path.exists(docrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + raise pagure.exceptions.RepoExistsException( + 'The docs repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository(docrepo, bare=True) + + ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], project.path) + if os.path.exists(ticketrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + shutil.rmtree(docrepo) + raise pagure.exceptions.RepoExistsException( + 'The tickets repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository( + ticketrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], project.path) + if os.path.exists(requestrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + shutil.rmtree(docrepo) + shutil.rmtree(ticketrepo) + raise pagure.exceptions.RepoExistsException( + 'The requests repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + # Install the default hook + plugin = pagure.lib.plugins.get_plugin('default') + dbobj = plugin.db_object() + dbobj.active = True + dbobj.project_id = project.id + session.add(dbobj) + session.flush() + plugin.set_up(project) + plugin.install(project, dbobj) + session.commit() session.remove() gc_clean() @@ -160,20 +163,22 @@ def update_git(name, namespace, user, ticketuid=None, requestuid=None): session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, name=name, - user=user, with_lock=True) - if ticketuid is not None: - obj = pagure.lib.get_issue_by_uid(session, ticketuid) - folder = APP.config['TICKETS_FOLDER'] - elif requestuid is not None: - obj = pagure.lib.get_request_by_uid(session, requestuid) - folder = APP.config['REQUESTS_FOLDER'] - else: - raise NotImplementedError('No ticket ID or request ID provided') + user=user) + with project.lock('WORKER'): + if ticketuid is not None: + obj = pagure.lib.get_issue_by_uid(session, ticketuid) + folder = APP.config['TICKETS_FOLDER'] + elif requestuid is not None: + obj = pagure.lib.get_request_by_uid(session, requestuid) + folder = APP.config['REQUESTS_FOLDER'] + else: + raise NotImplementedError('No ticket ID or request ID provided') + + if obj is None: + raise Exception('Unable to find object') + + result = pagure.lib.git._update_git(obj, project, folder) - if obj is None: - raise Exception('Unable to find object') - - result = pagure.lib.git._update_git(obj, project, folder) session.remove() gc_clean() return result @@ -184,14 +189,16 @@ def clean_git(name, namespace, user, ticketuid): session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, name=name, - user=user, with_lock=True) - obj = pagure.lib.get_issue_by_uid(session, ticketuid) - folder = APP.config['TICKETS_FOLDER'] + user=user) + with project.lock('WORKER'): + obj = pagure.lib.get_issue_by_uid(session, ticketuid) + folder = APP.config['TICKETS_FOLDER'] + + if obj is None: + raise Exception('Unable to find object') - if obj is None: - raise Exception('Unable to find object') + result = pagure.lib.git._clean_git(obj, project, folder) - result = pagure.lib.git._clean_git(obj, project, folder) session.remove() return result @@ -203,10 +210,11 @@ def update_file_in_git(name, namespace, user, branch, branchto, filename, userobj = pagure.lib.search_user(session, username=username) project = pagure.lib._get_project(session, namespace=namespace, name=name, - user=user, with_lock=True) + user=user) - pagure.lib.git._update_file_in_git(project, branch, branchto, filename, - content, message, userobj, email) + with project.lock('WORKER'): + pagure.lib.git._update_file_in_git(project, branch, branchto, filename, + content, message, userobj, email) session.remove() return ret('view_commits', repo=project.name, username=user, @@ -218,14 +226,15 @@ def delete_branch(name, namespace, user, branchname): session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, name=name, - user=user, with_lock=True) - repo_obj = pygit2.Repository(pagure.get_repo_path(project)) + user=user) + with project.lock('WORKER'): + repo_obj = pygit2.Repository(pagure.get_repo_path(project)) - try: - branch = repo_obj.lookup_branch(branchname) - branch.delete() - except pygit2.GitError as err: - _log.exception(err) + try: + branch = repo_obj.lookup_branch(branchname) + branch.delete() + except pygit2.GitError as err: + _log.exception(err) session.remove() return ret('view_repo', repo=name, namespace=namespace, username=user) @@ -238,66 +247,67 @@ def fork(name, namespace, user_owner, user_forker, editbranch, editfile): repo_from = pagure.lib._get_project(session, namespace=namespace, name=name, user=user_owner) repo_to = pagure.lib._get_project(session, namespace=namespace, name=name, - user=user_forker, with_lock=True) - - reponame = os.path.join(APP.config['GIT_FOLDER'], repo_from.path) - forkreponame = os.path.join(APP.config['GIT_FOLDER'], repo_to.path) - - frepo = pygit2.clone_repository(reponame, forkreponame, bare=True) - # Clone all the branches as well - for branch in frepo.listall_branches(pygit2.GIT_BRANCH_REMOTE): - branch_obj = frepo.lookup_branch(branch, pygit2.GIT_BRANCH_REMOTE) - branchname = branch_obj.branch_name.replace( - branch_obj.remote_name, '', 1)[1:] - if branchname in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): - continue - frepo.create_branch(branchname, frepo.get(branch_obj.target.hex)) - - # Create the git-daemon-export-ok file on the clone - http_clone_file = os.path.join(forkreponame, 'git-daemon-export-ok') - if not os.path.exists(http_clone_file): - with open(http_clone_file, 'w'): - pass - - docrepo = os.path.join(APP.config['DOCS_FOLDER'], repo_to.path) - if os.path.exists(docrepo): - shutil.rmtree(forkreponame) - raise pagure.exceptions.RepoExistsException( - 'The docs "%s" already exists' % repo_to.path - ) - pygit2.init_repository(docrepo, bare=True) - - ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], repo_to.path) - if os.path.exists(ticketrepo): - shutil.rmtree(forkreponame) - shutil.rmtree(docrepo) - raise pagure.exceptions.RepoExistsException( - 'The tickets repo "%s" already exists' % repo_to.path - ) - pygit2.init_repository( - ticketrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], repo_to.path) - if os.path.exists(requestrepo): - shutil.rmtree(forkreponame) - shutil.rmtree(docrepo) - shutil.rmtree(ticketrepo) - raise pagure.exceptions.RepoExistsException( - 'The requests repo "%s" already exists' % repo_to.path + user=user_forker) + + with repo_to.lock('WORKER'): + reponame = os.path.join(APP.config['GIT_FOLDER'], repo_from.path) + forkreponame = os.path.join(APP.config['GIT_FOLDER'], repo_to.path) + + frepo = pygit2.clone_repository(reponame, forkreponame, bare=True) + # Clone all the branches as well + for branch in frepo.listall_branches(pygit2.GIT_BRANCH_REMOTE): + branch_obj = frepo.lookup_branch(branch, pygit2.GIT_BRANCH_REMOTE) + branchname = branch_obj.branch_name.replace( + branch_obj.remote_name, '', 1)[1:] + if branchname in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): + continue + frepo.create_branch(branchname, frepo.get(branch_obj.target.hex)) + + # Create the git-daemon-export-ok file on the clone + http_clone_file = os.path.join(forkreponame, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w'): + pass + + docrepo = os.path.join(APP.config['DOCS_FOLDER'], repo_to.path) + if os.path.exists(docrepo): + shutil.rmtree(forkreponame) + raise pagure.exceptions.RepoExistsException( + 'The docs "%s" already exists' % repo_to.path + ) + pygit2.init_repository(docrepo, bare=True) + + ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], repo_to.path) + if os.path.exists(ticketrepo): + shutil.rmtree(forkreponame) + shutil.rmtree(docrepo) + raise pagure.exceptions.RepoExistsException( + 'The tickets repo "%s" already exists' % repo_to.path + ) + pygit2.init_repository( + ticketrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], repo_to.path) + if os.path.exists(requestrepo): + shutil.rmtree(forkreponame) + shutil.rmtree(docrepo) + shutil.rmtree(ticketrepo) + raise pagure.exceptions.RepoExistsException( + 'The requests repo "%s" already exists' % repo_to.path + ) + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + pagure.lib.notify.log( + repo_to, + topic='project.forked', + msg=dict( + project=repo_to.to_json(public=True), + agent=user_forker, + ), ) - pygit2.init_repository( - requestrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - pagure.lib.notify.log( - repo_to, - topic='project.forked', - msg=dict( - project=repo_to.to_json(public=True), - agent=user_forker, - ), - ) session.remove() del frepo @@ -343,12 +353,13 @@ def merge_pull_request(name, namespace, user, requestid, user_merger): session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, - name=name, user=user, with_lock=True) - request = pagure.lib.search_pull_requests( - session, project_id=project.id, requestid=requestid) + name=name, user=user) + with project.lock('WORKER'): + request = pagure.lib.search_pull_requests( + session, project_id=project.id, requestid=requestid) - pagure.lib.git.merge_pull_request( - session, request, user_merger, APP.config['REQUESTS_FOLDER']) + pagure.lib.git.merge_pull_request( + session, request, user_merger, APP.config['REQUESTS_FOLDER']) refresh_pr_cache.delay(name, namespace, user) session.remove() @@ -362,12 +373,35 @@ def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): project = pagure.lib._get_project(session, namespace=namespace, name=name, user=user) - issue = pagure.lib.get_issue_by_uid(session, issueuid) - user_attacher = pagure.lib.search_user(session, username=user_attacher) + with project.lock('WORKER'): + issue = pagure.lib.get_issue_by_uid(session, issueuid) + user_attacher = pagure.lib.search_user(session, username=user_attacher) - pagure.lib.git._add_file_to_git( - project, issue, APP.config['ATTACHMENTS_FOLDER'], - APP.config['TICKETS_FOLDER'], user_attacher, filename) + pagure.lib.git._add_file_to_git( + project, issue, APP.config['ATTACHMENTS_FOLDER'], + APP.config['TICKETS_FOLDER'], user_attacher, filename) session.remove() gc_clean() + + +@conn.task +def project_dowait(name, namespace, user): + """ This is a task used to test the locking systems. + + It should never be allowed to be called in production instances, since that + would allow an attacker to basically DOS a project by calling this + repeatedly. """ + assert APP.config.get('ALLOW_PROJECT_DOWAIT', False) + + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user) + with project.lock('WORKER'): + time.sleep(10) + + session.remove() + gc_clean() + + return ret('view_repo', repo=name, username=user, namespace=namespace) diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 42aef77..9b8ed55 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -2611,3 +2611,26 @@ def give_project(repo, username=None, namespace=None): return flask.redirect(flask.url_for( 'view_repo', username=username, repo=repo.name, namespace=namespace)) + + +@APP.route('//dowait/') +@APP.route('//dowait') +@APP.route('///dowait/') +@APP.route('///dowait') +@APP.route('/fork///dowait/') +@APP.route('/fork///dowait') +@APP.route('/fork////dowait/') +@APP.route('/fork////dowait') +def project_dowait(repo, username=None, namespace=None): + """ Schedules a task that just waits 10 seconds for testing locking. + + This is not available unless ALLOW_PROJECT_DOWAIT is set to True, which + should only ever be done in test instances. + """ + if not APP.config.get('ALLOW_PROJECT_DOWAIT', False): + flask.abort(401, 'No') + + taskid = pagure.lib.tasks.project_dowait.delay( + name=repo, namespace=namespace, user=username).id + + return pagure.wait_for_task(taskid) diff --git a/tests/__init__.py b/tests/__init__.py index 1ffa549..3b45dcb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -64,6 +64,7 @@ REQUESTS_FOLDER = '%(path)s/requests' REMOTE_GIT_FOLDER = '%(path)s/remotes' ATTACHMENTS_FOLDER = '%(path)s/attachments' DB_URL = '%(dburl)s' +ALLOW_PROJECT_DOWAIT = True """ diff --git a/tests/test_config b/tests/test_config index d52090c..04970a4 100644 --- a/tests/test_config +++ b/tests/test_config @@ -1 +1,2 @@ PAGURE_CI_SERVICES = ['jenkins'] +ALLOW_PROJECT_DOWAIT = True