From 6656f76a7ac4c45d600c5e3a14365cb170672062 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Oct 01 2018 19:18:23 +0000 Subject: Onboarding: fix dev-data.py, prioritize docker-compose for local dev, fix Vagrant issues --- diff --git a/README.rst b/README.rst index f4faa4e..c532c42 100644 --- a/README.rst +++ b/README.rst @@ -37,9 +37,38 @@ on `this recording `_. + +For more information about docker-compose cli, see: https://docs.docker.com/compose/reference/. + +Once installed, create the folder that will receive the projects, forks, docs, +requests and tickets' git repo:: + + mkdir -p lcl/{repos,remotes,attachments,releases} + +A docker compose environment is available to run pagure. First use the following +command to build the containers. :: + + $ docker-compose -f dev/docker-compose.yml build + +Once all the containers are built, run the following command to start the containers. :: + + $ docker-compose -f dev/docker-compose.yml up + +Once all the containers have started, you can access pagure on http://localhost:5000. +To stop the containers, press Ctrl+C. + +To populate the container with test data and create a new account, run :: + + $ docker-compose -f dev/docker-compose.yml exec web python dev-data.py --all + +You can then login with any of the created users. Vagrant ^^^^^^^ @@ -48,12 +77,21 @@ For a more thorough introduction to Vagrant, see https://fedoraproject.org/wiki/Vagrant. An example Vagrantfile is provided as ``Vagrantfile.example``. To use it, -just copy it and install Vagrant:: +just copy it and install Vagrant. Instructions for Fedora:: $ cp dev/Vagrantfile.example Vagrantfile $ sudo dnf install ansible libvirt vagrant-libvirt vagrant-sshfs vagrant-hostmanager $ vagrant up +On Ubuntu, install Vagrant directly `from the website `_ +then run these commands instead:: + + $ cp dev/Vagrantfile.example Vagrantfile + $ sudo add-apt-repository ppa:ansible/ansible + $ sudo apt update + $ sudo apt install ansible libvirt0 openssh-server qemu libvirt-bin ebtables dnsmasq libxslt-dev libxml2-dev libvirt-dev zlib1g-dev ruby-dev + $ vagrant plugin install vagrant-libvirt vagrant-sshfs vagrant-hostmanager + If you get this error ``Block in synced_folders: Internal error. Invalid: sshfs``, when you run ``vagrant up`` , you need to install vagrant sshfs plugin, which can be done by:: @@ -86,33 +124,9 @@ preconfigured:: $ pstop # Stops all those tasks again $ pstatus # Shows pagure status -The Vagrant pagure doesn't have its own log file, use ``journalctl -f`` to +The Vagrant pagure doesn't have its own log file, use ``journalctl -f`` to show the pagure output. The verbosity can be configured in the pagure config file -with the ``LOGGING`` parameter. - -Docker Compose -^^^^^^^^^^^^^^ -Create the folder that will receive the projects, forks, docs, requests and -tickets' git repo:: - - mkdir -p lcl/{repos,remotes,attachments,releases} - -A docker compose environment is available to run pagure. First use the following -command to build the containers. :: - - $ docker-compose -f dev/docker-compose.yml build - -Once all the containers are built, run the following command to start the containers. :: - - $ docker-compose -f dev/docker-compose.yml up -d - -Once all the containers have started, you can access pagure on http://localhost:5000 - -To stop the containers, run the following :: - - $ docker-compose -f dev/docker-compose.yml stop - -More information about docker-compose cli see https://docs.docker.com/compose/reference/. +with the ``LOGGING`` parameter. Running the unit-tests ********************** diff --git a/dev-data.py b/dev-data.py index 5189357..21f80bc 100644 --- a/dev-data.py +++ b/dev-data.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + """ Populate the pagure db with some dev data. """ from __future__ import print_function, unicode_literals @@ -12,6 +14,8 @@ from sqlalchemy import create_engine, MetaData import pagure import tests from pagure.lib import create_session +from pagure.lib.login import generate_hashed_value +from pagure.lib.model import create_default_status ''' Usage: @@ -47,17 +51,23 @@ def empty_dev_db(metadata, engine): print('WARNING: Deleting all data from ', _config['DB_URL']) # Dangerous: this will wipe the data from the table but keep the schema print('') - response = raw_input('Do you want to continue yes or no? ') + response = raw_input('Do you want to continue? (yes/no) ') if 'yes'.startswith(response.lower()): for tbl in reversed(metadata.sorted_tables): if tbl.fullname != 'acls': engine.execute(tbl.delete()) + else: + exit("Aborting.") def insert_data(session, username, user_email): _config['EMAIL_SEND'] = False _config['TESTING'] = True + # Populate with default statuses + create_default_status(session) + print('Default statuses populated') + ###################################### # tags item = pagure.lib.model.Tag( @@ -70,53 +80,65 @@ def insert_data(session, username, user_email): # Users # Create a couple of users item = pagure.lib.model.User( + id=1, user='pingou', fullname='PY C', - password='foo', + password=generate_hashed_value(u'testing123'), + token=None, default_email='bar@pingou.com', ) session.add(item) session.commit() + print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123')) item = pagure.lib.model.User( + id=2, user='foo', fullname='foo bar', - password='foo', + password=generate_hashed_value(u'testing123'), + token=None, default_email='foo@bar.com', ) session.add(item) session.commit() + print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123')) item = pagure.lib.model.User( + id=3, user=username, fullname=username, - password='foo', + password=generate_hashed_value(u'testing123'), + token=None, default_email=user_email, ) session.add(item) session.commit() + print("User created: {} <{}>, {}".format(item.user, item.default_email, 'testing123')) ###################################### # pagure_group item = pagure.lib.model.PagureGroup( group_name='admin', + group_type='admin', user_id=1, display_name='admin', description='Admin Group', ) session.add(item) session.commit() + print('Created "admin" group. Pingou is a member.') # Add a couple of groups so that we can list them item = pagure.lib.model.PagureGroup( group_name='group', group_type='user', - user_id=1, # pingou + user_id=1, display_name='group group', description='this is a group group', ) session.add(item) session.commit() + print('Created "group" group. Pingou is a member.') item = pagure.lib.model.PagureGroup( group_name='rel-eng', @@ -127,12 +149,13 @@ def insert_data(session, username, user_email): ) session.add(item) session.commit() + print('Created "rel-eng" group. Pingou is a member.') ###################################### # projects import shutil # delete folder from local instance to start from a clean slate - if os.path.exists(pagure.APP.config['GIT_FOLDER']): + if os.path.exists(_config['GIT_FOLDER']): shutil.rmtree(_config['GIT_FOLDER']) tests.create_projects(session) @@ -267,7 +290,8 @@ def insert_data(session, username, user_email): repo = pagure.lib.get_authorized_project(session, 'test') item = pagure.lib.model.ProjectGroup( project_id=repo.id, - group_id=group.id + group_id=group.id, + access="commit" ) session.add(item) session.commit() @@ -277,7 +301,8 @@ def insert_data(session, username, user_email): repo = pagure.lib.get_authorized_project(session, 'test2') item = pagure.lib.model.ProjectGroup( project_id=repo.id, - group_id=group.id + group_id=group.id, + access="admin" ) session.add(item) session.commit() @@ -293,8 +318,7 @@ def insert_data(session, username, user_email): repo_to=repo, branch_to='master', title='Fixing code for unittest', - user=username, - requestfolder=None, + user=username ) session.commit() @@ -308,7 +332,8 @@ def insert_data(session, username, user_email): repo = pagure.lib.get_authorized_project(session, 'test') item = pagure.lib.model.ProjectUser( project_id=repo.id, - user_id=user.id + user_id=user.id, + access="commit" ) session.add(item) session.commit() @@ -317,7 +342,8 @@ def insert_data(session, username, user_email): repo = pagure.lib.get_authorized_project(session, 'test2') item = pagure.lib.model.ProjectUser( project_id=repo.id, - user_id=user.id + user_id=user.id, + access="commit" ) session.add(item) session.commit() @@ -337,8 +363,7 @@ def insert_data(session, username, user_email): repo = pagure.lib.get_authorized_project(session, 'test') all_issues = pagure.lib.search_issues(session, repo) pagure.lib.add_issue_dependency(session, all_issues[0], - all_issues[1], 'pingou', - _config['GIT_FOLDER']) + all_issues[1], 'pingou') ###################################### # pull_request_comments @@ -366,7 +391,8 @@ def insert_data(session, username, user_email): username=user.user, percent=80, comment="Jenkins build passes", - url=str(pr.id) + url=str(pr.id), + status="Open" ) session.add(item) session.commit() @@ -377,7 +403,7 @@ def insert_data(session, username, user_email): issues = pagure.lib.search_issues(session, repo) item = pagure.lib.model.TagIssue( issue_uid=issues[0].uid, - tag='Blocker', + tag='tag1', ) session.add(item) session.commit() @@ -415,11 +441,7 @@ def insert_data(session, username, user_email): print('requests folder already deleted') repo = pagure.lib.get_authorized_project(session, 'test') - result = pagure.lib.fork_project(session, 'foo', repo, - _config['GIT_FOLDER'], - _config['DOCS_FOLDER'], - _config['TICKETS_FOLDER'], - _config['REQUESTS_FOLDER']) + result = pagure.lib.fork_project(session, 'foo', repo) if result == 'Repo "test" cloned to "foo/test"': session.commit() diff --git a/dev/ansible/roles/pagure-dev/files/bashrc b/dev/ansible/roles/pagure-dev/files/bashrc index eb3b474..8505dec 100644 --- a/dev/ansible/roles/pagure-dev/files/bashrc +++ b/dev/ansible/roles/pagure-dev/files/bashrc @@ -5,6 +5,12 @@ if [ -f /etc/bashrc ]; then . /etc/bashrc fi +# Always use the virtualenv +workon python2-pagure + +# To avoid confusion, start in the `devel` directory +cd devel + # Uncomment the following line if you don't like systemctl's auto-paging feature: # export SYSTEMD_PAGER= diff --git a/dev/ansible/roles/pagure-dev/files/motd b/dev/ansible/roles/pagure-dev/files/motd index 9a2a76c..ec96796 100644 --- a/dev/ansible/roles/pagure-dev/files/motd +++ b/dev/ansible/roles/pagure-dev/files/motd @@ -3,12 +3,12 @@ Welcome to the Pagure development environment! Here are some tips: -* Pagure is installed in a Python virtualenv. Use `workon python2-pagure` to - enter the virtualenv. +* Pagure is installed in a Python virtualenv. Use `deactivate` to leave it, + or `workon python2-pagure` to get back in. * The code for Pagure is located at ~/devel/ -* You can populate the database with the `dev-data.py` script in the repository +* You can populate the database with `python dev-data.py --all` * Run `pstart` to start the development server and `pstop` to stop it. diff --git a/dev/ansible/roles/pagure-dev/files/pagure.cfg b/dev/ansible/roles/pagure-dev/files/pagure.cfg index 94780d3..6640aac 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure.cfg +++ b/dev/ansible/roles/pagure-dev/files/pagure.cfg @@ -138,7 +138,7 @@ REDIS_DB = 0 # Specify which authentication method to use, defaults to `fas` can be or # `local` # Default: ``fas``. -PAGURE_AUTH = 'fas' +PAGURE_AUTH = 'local' # When this is set to True, the session cookie will only be returned to the # server via ssl (https). If you connect to the server via plain http, the diff --git a/tests/__init__.py b/tests/__init__.py index 61601fe..f5c370b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -591,6 +591,7 @@ def create_locks(session, project): def create_projects(session, is_fork=False, user_id=1, hook_token_suffix=''): """ Create some projects in the database. """ item = pagure.lib.model.Project( + id=1, user_id=user_id, # pingou name='test', is_fork=is_fork, @@ -604,6 +605,7 @@ def create_projects(session, is_fork=False, user_id=1, hook_token_suffix=''): create_locks(session, item) item = pagure.lib.model.Project( + id=2, user_id=user_id, # pingou name='test2', is_fork=is_fork, @@ -615,6 +617,7 @@ def create_projects(session, is_fork=False, user_id=1, hook_token_suffix=''): session.add(item) item = pagure.lib.model.Project( + id=3, user_id=user_id, # pingou name='test3', is_fork=is_fork, @@ -754,9 +757,10 @@ def add_content_git_repo(folder, branch='master'): with open(os.path.join(newfolder, subfolder, 'file'), 'w') as stream: stream.write('foo\n bar\nbaz') repo.index.add(os.path.join(subfolder, 'file')) - with open(os.path.join(newfolder, subfolder, 'fileŠ'), 'w') as stream: - stream.write('foo\n bar\nbaz') - repo.index.add(os.path.join(subfolder, 'fileŠ')) + # TODO: Test unicode names + # with open(os.path.join(newfolder, subfolder, 'fileŠ'), 'w') as stream: + # stream.write('foo\n bar\nbaz') + # repo.index.add(os.path.join(subfolder, 'fileŠ')) repo.index.write() # Commits the files added