#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Populate the pagure db with some dev data. """
from __future__ import print_function, unicode_literals, absolute_import
import argparse
import os
import tempfile
import pygit2
import shutil
import six
from sqlalchemy import create_engine, MetaData
import pagure
import tests
import pagure.lib.model
import pagure.lib.query
from pagure.lib.login import generate_hashed_value
from pagure.lib.model import create_default_status
from pagure.lib.repo import PagureRepo
'''
Usage:
python dev-data.py --init
python dev-data.py --clean
python dev-data.py --populate
python dev-data.py --all
'''
_config = pagure.config.reload_config()
def empty_dev_db(session):
print('')
print('WARNING: Deleting all data from', _config['DB_URL'])
response = os.environ.get("FORCE_DELETE")
if not response:
response = six.moves.input('Do you want to continue? (yes/no) ')
if response.lower().startswith('y'):
tables = reversed(pagure.lib.model_base.BASE.metadata.sorted_tables)
for tbl in tables:
session.execute(tbl.delete())
else:
exit("Aborting.")
def insert_data(session, username, user_email):
_config['EMAIL_SEND'] = False
_config['TESTING'] = True
######################################
# tags
item = pagure.lib.model.Tag(
tag='tag1',
)
session.add(item)
session.commit()
######################################
# Users
# Create a couple of users
pingou = item = pagure.lib.model.User(
user='pingou',
fullname='PY C',
password=generate_hashed_value(u'testing123'),
token=None,
default_email='bar@pingou.com',
)
session.add(item)
session.commit()
print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123'))
foo = item = pagure.lib.model.User(
user='foo',
fullname='foo bar',
password=generate_hashed_value(u'testing123'),
token=None,
default_email='foo@bar.com',
)
session.add(item)
session.commit()
print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123'))
you = item = pagure.lib.model.User(
user=username,
fullname=username,
password=generate_hashed_value(u'testing123'),
token=None,
default_email=user_email,
)
session.add(item)
session.commit()
print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123'))
######################################
# pagure_group
item = pagure.lib.model.PagureGroup(
group_name='admin',
group_type='admin',
user_id=pingou.id,
display_name='admin',
description='Admin Group',
)
session.add(item)
session.commit()
print('Created "admin" group. Pingou is a member.')
# Add a couple of groups so that we can list them
item = pagure.lib.model.PagureGroup(
group_name='group',
group_type='user',
user_id=pingou.id,
display_name='group group',
description='this is a group group',
)
session.add(item)
session.commit()
print('Created "group" group. Pingou is a member.')
item = pagure.lib.model.PagureGroup(
group_name='rel-eng',
group_type='user',
user_id=pingou.id,
display_name='Release Engineering',
description='The group of release engineers',
)
session.add(item)
session.commit()
print('Created "rel-eng" group. Pingou is a member.')
######################################
# projects
import shutil
# delete folder from local instance to start from a clean slate
if os.path.exists(_config['GIT_FOLDER']):
shutil.rmtree(_config['GIT_FOLDER'])
# Create projects
item = project1 = pagure.lib.model.Project(
user_id=pingou.id,
name='test',
is_fork=False,
parent_id=None,
description='test project #1',
hook_token='aaabbbccc',
)
item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']
session.add(item)
session.flush()
tests.create_locks(session, item)
item = project2 = pagure.lib.model.Project(
user_id=pingou.id,
name='test2',
is_fork=False,
parent_id=None,
description='test project #2',
hook_token='aaabbbddd',
)
item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']
session.add(item)
item = project3 = pagure.lib.model.Project(
user_id=pingou.id,
name='test3',
is_fork=False,
parent_id=None,
description='namespaced test project',
hook_token='aaabbbeee',
namespace='somenamespace',
)
item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']
session.add(item)
session.commit()
tests.create_projects_git(_config['GIT_FOLDER'], bare=True)
add_content_git_repo(
os.path.join(_config['GIT_FOLDER'], 'test.git'))
tests.add_readme_git_repo(
os.path.join(_config['GIT_FOLDER'], 'test.git'))
# Add some content to the git repo
add_content_git_repo(
os.path.join(_config['GIT_FOLDER'], 'forks', 'pingou',
'test.git'))
tests.add_readme_git_repo(
os.path.join(_config['GIT_FOLDER'], 'forks', 'pingou',
'test.git'))
tests.add_commit_git_repo(
os.path.join(_config['GIT_FOLDER'], 'forks', 'pingou',
'test.git'), ncommits=10)
######################################
# user_emails
item = pagure.lib.model.UserEmail(
user_id=pingou.id,
email='bar@pingou.com')
session.add(item)
item = pagure.lib.model.UserEmail(
user_id=pingou.id,
email='foo@pingou.com')
session.add(item)
item = pagure.lib.model.UserEmail(
user_id=foo.id,
email='foo@bar.com')
session.add(item)
item = pagure.lib.model.UserEmail(
user_id=you.id,
email=user_email)
session.add(item)
session.commit()
######################################
# user_emails_pending
email_pend = pagure.lib.model.UserEmailPending(
user_id=pingou.id,
email='foo@fp.o',
token='abcdef',
)
session.add(email_pend)
session.commit()
######################################
# issues
# Add an issue and tag it so that we can list them
item = pagure.lib.model.Issue(
id=1001,
uid='foobar',
project_id=project1.id,
title='Problem with jenkins build',
content='For some reason the tests fail at line:24',
user_id=pingou.id,
)
session.add(item)
session.commit()
item = pagure.lib.model.Issue(
id=1002,
uid='foobar2',
project_id=project1.id,
title='Unit tests failing',
content='Need to fix code for the unit tests to '
'pass so jenkins build can complete.',
user_id=pingou.id,
)
session.add(item)
session.commit()
item = pagure.lib.model.Issue(
id=1003,
uid='foobar3',
project_id=project1.id,
title='Segfault during execution',
content='Index out of bounds for variable i?',
user_id=you.id,
)
session.add(item)
session.commit()
######################################
# pagure_user_group
group = pagure.lib.query.search_groups(session, pattern=None,
group_name="rel-eng", group_type=None)
item = pagure.lib.model.PagureUserGroup(
user_id=pingou.id,
group_id=group.id
)
session.add(item)
session.commit()
group = pagure.lib.query.search_groups(session, pattern=None,
group_name="admin", group_type=None)
item = pagure.lib.model.PagureUserGroup(
user_id=you.id,
group_id=group.id
)
session.add(item)
session.commit()
group = pagure.lib.query.search_groups(session, pattern=None,
group_name="group", group_type=None)
item = pagure.lib.model.PagureUserGroup(
user_id=foo.id,
group_id=group.id
)
session.add(item)
session.commit()
######################################
# projects_groups
group = pagure.lib.query.search_groups(session, pattern=None,
group_name="rel-eng", group_type=None)
repo = pagure.lib.query.get_authorized_project(session, 'test')
item = pagure.lib.model.ProjectGroup(
project_id=repo.id,
group_id=group.id,
access="commit"
)
session.add(item)
session.commit()
group = pagure.lib.query.search_groups(session, pattern=None,
group_name="admin", group_type=None)
repo = pagure.lib.query.get_authorized_project(session, 'test2')
item = pagure.lib.model.ProjectGroup(
project_id=repo.id,
group_id=group.id,
access="admin"
)
session.add(item)
session.commit()
######################################
# pull_requests
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='Fixing code for unittest',
user=username,
status="Open"
)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='add very nice README',
user=username,
status="Open"
)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='Add README',
user=username,
status="Closed"
)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='Fix some containers',
user=username,
status="Merged"
)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='Fix pull request statuses',
user=username,
status="Closed"
)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test')
forked_repo = pagure.lib.query.get_authorized_project(session, 'test')
req = pagure.lib.query.new_pull_request(
session=session,
repo_from=forked_repo,
branch_from='master',
repo_to=repo,
branch_to='master',
title='Fixing UI of issue',
user=username,
status="Merged"
)
session.commit()
#####################################
# tokens
tests.create_tokens(session, user_id=pingou.id, project_id=project1.id)
######################################
# user_projects
repo = pagure.lib.query.get_authorized_project(session, 'test')
item = pagure.lib.model.ProjectUser(
project_id=repo.id,
user_id=foo.id,
access="commit"
)
session.add(item)
session.commit()
repo = pagure.lib.query.get_authorized_project(session, 'test2')
item = pagure.lib.model.ProjectUser(
project_id=repo.id,
user_id=you.id,
access="commit"
)
session.add(item)
session.commit()
######################################
# issue_comments
item = pagure.lib.model.IssueComment(
user_id=pingou.id,
issue_uid='foobar',
comment='We may need to adjust the unittests instead of the code.',
)
session.add(item)
session.commit()
######################################
# issue_to_issue
repo = pagure.lib.query.get_authorized_project(session, 'test')
all_issues = pagure.lib.query.search_issues(session, repo)
pagure.lib.query.add_issue_dependency(session, all_issues[0],
all_issues[1], 'pingou')
######################################
# pull_request_comments
user = pagure.lib.query.search_user(session, username='pingou')
# only 1 pull request available atm
pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0]
item = pagure.lib.model.PullRequestComment(
pull_request_uid=pr.uid,
user_id=user.id,
comment="+1 for me. Btw, could you rebase before you merge?",
notification=0
)
session.add(item)
session.commit()
######################################
# pull_request_flags
# only 1 pull request available atm
pr = pagure.lib.query.get_pull_request_of_user(session, "pingou")[0]
item = pagure.lib.model.PullRequestFlag(
uid="random_pr_flag_uid",
pull_request_uid=pr.uid,
user_id=pingou.id,
username=pingou.user,
percent=80,
comment="Jenkins build passes",
url=str(pr.id),
status="success"
)
session.add(item)
session.commit()
pr = pagure.lib.query.get_pull_request_of_user(session, "foo")[1]
item = pagure.lib.model.PullRequestFlag(
uid="oink oink uid",
pull_request_uid=pr.uid,
user_id=pingou.id,
username=pingou.user,
percent=80,
comment="Jenkins does not pass",
url=str(pr.id),
status="failure"
)
session.add(item)
session.commit()
######################################
# pull_request_assignee
pr = pagure.lib.query.search_pull_requests(session, requestid='1006')
pr.assignee_id = pingou.id
session.commit()
pr = pagure.lib.query.search_pull_requests(session, requestid='1007')
pr.assignee_id = you.id
session.commit()
pr = pagure.lib.query.search_pull_requests(session, requestid='1004')
pr.assignee_id = foo.id
session.commit()
######################################
# tags_issues
repo = pagure.lib.query.get_authorized_project(session, 'test')
issues = pagure.lib.query.search_issues(session, repo)
item = pagure.lib.model.TagIssue(
issue_uid=issues[0].uid,
tag='tag1',
)
session.add(item)
session.commit()
######################################
# tokens_acls
tests.create_tokens_acl(session)
######################################
# Fork a project
# delete fork data
fork_proj_location = "forks/foo/test.git"
try:
shutil.rmtree(os.path.join(_config['GIT_FOLDER'],
fork_proj_location))
except:
print('git folder already deleted')
try:
shutil.rmtree(os.path.join(_config['DOCS_FOLDER'],
fork_proj_location))
except:
print('docs folder already deleted')
try:
shutil.rmtree(os.path.join(_config['TICKETS_FOLDER'],
fork_proj_location))
except:
print('tickets folder already deleted')
try:
shutil.rmtree(os.path.join(_config['REQUESTS_FOLDER'],
fork_proj_location))
except:
print('requests folder already deleted')
repo = pagure.lib.query.get_authorized_project(session, 'test')
result = pagure.lib.query.fork_project(session, 'foo', repo)
if result == 'Repo "test" cloned to "foo/test"':
session.commit()
def add_content_git_repo(folder, branch='master'):
""" Create some content for the specified git repo. """
if not os.path.exists(folder):
os.makedirs(folder)
brepo = pygit2.init_repository(folder, bare=True)
newfolder = tempfile.mkdtemp(prefix='pagure-tests')
repo = pygit2.clone_repository(folder, newfolder)
# Create a file in that git repo
with open(os.path.join(newfolder, 'sources'), 'w') as stream:
stream.write('foo\n bar')
repo.index.add('sources')
repo.index.write()
parents = []
commit = None
try:
commit = repo.revparse_single(
'HEAD' if branch == 'master' else branch)
except KeyError:
pass
if commit:
parents = [commit.oid.hex]
# Commits the files added
tree = repo.index.write_tree()
author = pygit2.Signature(
'Alice Author', 'alice@authors.tld')
committer = pygit2.Signature(
'Cecil Committer', 'cecil@committers.tld')
repo.create_commit(
'refs/heads/%s' % branch, # the name of the reference to update
author,
committer,
'Add sources file for testing',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
parents,
)
parents = []
commit = None
try:
commit = repo.revparse_single(
'HEAD' if branch == 'master' else branch)
except KeyError:
pass
if commit:
parents = [commit.oid.hex]
subfolder = os.path.join('folder1', 'folder2')
if not os.path.exists(os.path.join(newfolder, subfolder)):
os.makedirs(os.path.join(newfolder, subfolder))
# Create a file in that git repo
with open(os.path.join(newfolder, subfolder, 'file'), 'w') as stream:
stream.write('foo\n bar\nbaz')
repo.index.add(os.path.join(subfolder, 'file'))
repo.index.write()
# Commits the files added
tree = repo.index.write_tree()
author = pygit2.Signature(
'Alice Author', 'alice@authors.tld')
committer = pygit2.Signature(
'Cecil Committer', 'cecil@committers.tld')
repo.create_commit(
'refs/heads/%s' % branch, # the name of the reference to update
author,
committer,
'Add some directory and a file for more testing',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
parents
)
# Push to origin
ori_remote = repo.remotes[0]
master_ref = repo.lookup_reference(
'HEAD' if branch == 'master' else 'refs/heads/%s' % branch).resolve()
refname = '%s:%s' % (master_ref.name, master_ref.name)
PagureRepo.push(ori_remote, refname)
shutil.rmtree(newfolder)
def _get_username():
invalid_option = ['pingou', 'foo']
user_name = os.environ.get("USER_NAME")
if not user_name:
print("")
user_name = six.moves.input(
"Enter your username so we can add you into the test data: ")
cnt = 0
while not user_name.strip() or user_name in invalid_option:
print("Reserved names: " + str(invalid_option))
user_name = six.moves.input(
"Enter your username so we can add you into the "
"test data: ")
cnt += 1
if cnt == 4:
print("We asked too many times, bailing")
sys.exit(1)
return user_name
def _get_user_email():
invalid_option = ['bar@pingou.com', 'foo@bar.com']
user_email = os.environ.get("USER_EMAIL")
if not user_email:
print("")
user_email = six.moves.input("Enter your user email: ")
cnt = 0
while not user_email.strip() or user_email in invalid_option:
print("Reserved names: " + str(invalid_option))
user_email = six.moves.input("Enter your user email: ")
cnt += 1
if cnt == 4:
print("We asked too many times, bailing")
sys.exit(1)
return user_email
if __name__ == "__main__":
desc = "Run the dev database initialization/insertion/deletion " \
"script for db located " + str(_config['DB_URL'])
parser = argparse.ArgumentParser(prog="dev-data", description=desc)
parser.add_argument('-i', '--init', action="store_true",
help="Create the dev db")
parser.add_argument('-p', '--populate', action="store_true",
help="Add test data to the db")
parser.add_argument('-d', '--delete', action="store_true",
help="Wipe the dev db")
parser.add_argument('-a', '--all', action="store_true",
help="Create, Populate then Wipe the dev db")
args = parser.parse_args()
# forcing the user to choose
if not any(vars(args).values()):
parser.error('No arguments provided.')
session = None
if args.init or args.all:
session = pagure.lib.model.create_tables(
db_url=_config["DB_URL"],
alembic_ini=None,
acls=_config["ACLS"],
debug=False)
print("Database created")
if args.populate or args.all:
if not session:
session = pagure.lib.query.create_session(_config['DB_URL'])
user_name = _get_username()
user_email = _get_user_email()
insert_data(session, user_name, user_email)
if args.delete or args.all:
empty_dev_db(session)