diff --git a/pagure/__init__.py b/pagure/__init__.py index fcb673c..42a5012 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -18,6 +18,7 @@ __api_version__ = '0.12' import datetime # noqa: E402 import logging # noqa: E402 +import logging.config # noqa: E402 import os # noqa: E402 import re # noqa: E402 import urlparse # noqa: E402 @@ -38,7 +39,7 @@ else: import pagure.exceptions # noqa: E402 # Create the application. -APP = MultiStaticFlask(__name__) +APP = MultiStaticFlask('pagure') if perfrepo: # Do this as early as possible. @@ -55,6 +56,8 @@ APP.config.from_object('pagure.default_config') if 'PAGURE_CONFIG' in os.environ: APP.config.from_envvar('PAGURE_CONFIG') +logging.config.dictConfig(APP.config.get('LOGGING') or {'version': 1}) + if APP.config.get('THEME_TEMPLATE_FOLDER', False): # Jinja can be told to look for templates in different folders @@ -145,7 +148,7 @@ if APP.config.get('PAGURE_AUTH', None) in ['fas', 'openid']: is_admin=is_admin(), ) except pagure.exceptions.PagureException as err: - LOG.debug(err) + APP.logger.debug(err) # Remove the old groups for group in groups - fas_groups: try: @@ -158,13 +161,12 @@ if APP.config.get('PAGURE_AUTH', None) in ['fas', 'openid']: force=True, ) except pagure.exceptions.PagureException as err: - LOG.debug(err) + APP.logger.debug(err) SESSION.commit() except SQLAlchemyError as err: SESSION.rollback() - LOG.debug(err) - LOG.exception(err) + APP.logger.exception(err) flask.flash( 'Could not set up you as a user properly, please contact ' 'an admin', 'error') @@ -197,14 +199,6 @@ if not APP.debug: from_email=APP.config.get('FROM_EMAIL', 'pagure@fedoraproject.org') )) -# Send classic logs into syslog -SHANDLER = logging.StreamHandler() -SHANDLER.setLevel(APP.config.get('LOG_LEVEL', 'INFO')) -APP.logger.addHandler(SHANDLER) - -LOG = APP.logger -LOG.setLevel(APP.config.get('LOG_LEVEL', 'INFO')) -pagure.lib.set_log(LOG) APP.wsgi_app = pagure.proxy.ReverseProxied(APP.wsgi_app) @@ -689,7 +683,7 @@ def get_remote_repo_path(remote_git, branch_from, loop=False): pygit2.clone_repository( remote_git, repopath, checkout_branch=branch_from) except Exception as err: - LOG.exception(err) + APP.logger.exception(err) flask.abort( 500, 'The following error was raised when trying to clone the ' @@ -700,16 +694,17 @@ def get_remote_repo_path(remote_git, branch_from, loop=False): try: repo.pull(branch=branch_from, force=True) except pygit2.GitError as err: - LOG.debug('Error pull the repo: %s -- error: %s' % (repopath, err)) + APP.logger.debug( + 'Error pull the repo: %s -- error: %s' % (repopath, err)) if str(err).lower() != 'no content-type header in response': - LOG.exception(err) + APP.logger.exception(err) flask.abort( 500, 'The following error was raised when trying to pull the ' 'changes from the remote: %s' % str(err) ) except pagure.exceptions.PagureException as err: - LOG.exception(err) + APP.logger.exception(err) flask.abort(500, str(err)) return repopath diff --git a/pagure/default_config.py b/pagure/default_config.py index 4501d19..b151e2b 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -258,3 +258,45 @@ USER_NAMESPACE = False EXCLUDE_GROUP_INDEX = [] TRIGGER_CI = ['pretty please pagure-ci rebuild'] + + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + }, + }, + 'handlers': { + 'console': { + 'level': 'INFO', + 'formatter': 'standard', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', + }, + }, + # The root logger configuration; this is a catch-all configuration + # that applies to all log messages not handled by a different logger + 'root': { + 'level': 'INFO', + 'handlers': ['console'], + }, + 'loggers': { + 'pagure': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': False + }, + 'flask': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': False + }, + 'sqlalchemy': { + 'handlers': ['console'], + 'level': 'WARN', + 'propagate': False + }, + } +} diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index ef279c4..6753843 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -22,6 +22,7 @@ except ImportError: import json import datetime +import logging import markdown import os import shutil @@ -56,7 +57,7 @@ from pagure.lib import model REDIS = None PAGURE_CI = None -LOG = None +_log = logging.getLogger(__name__) def set_redis(host, port, dbname): @@ -72,12 +73,6 @@ def set_pagure_ci(services): PAGURE_CI = services -def set_log(logger): - """ Set a logger that can be used in this module. """ - global LOG - LOG = logger - - def get_user(session, key): """ Searches for a user in the database for a given username or email. """ @@ -3400,7 +3395,7 @@ def text2markdown(text, extended=True, readme=False): try: text = md_processor.convert(text) except Exception: - LOG.debug( + _log.debug( 'A markdown error occured while processing: ``%s``', str(text)) return clean_input(text) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index d4f7e00..4831a7f 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -17,6 +17,7 @@ import datetime import hashlib import json +import logging import os import shutil import subprocess @@ -36,6 +37,9 @@ from pagure.lib import model from pagure.lib.repo import PagureRepo +_log = logging.getLogger(__name__) + + def commit_to_patch(repo_obj, commits): ''' For a given commit (PyGit2 commit object) of a specified git repo, returns a string representation of the changes the commit did in a @@ -88,6 +92,7 @@ def write_gitolite_acls(session, configfile): ''' Generate the configuration file for gitolite for all projects on the forge. ''' + _log.info('Write down the gitolite configuration file') global_pr_only = pagure.APP.config.get('PR_ONLY', False) config = [] groups = {} @@ -97,6 +102,7 @@ def write_gitolite_acls(session, configfile): model.Project.id ) for project in query.all(): + _log.debug(' Processing project: %s', project.fullname) for group in project.committer_groups: if group.group_name not in groups: groups[group.group_name] = [ @@ -154,6 +160,7 @@ def _get_gitolite_command(): """ Return the gitolite command to run based on the info in the configuration file. """ + _log.info('Compiling the gitolite configuration') gitolite_folder = pagure.APP.config.get('GITOLITE_HOME', None) gitolite_version = pagure.APP.config.get('GITOLITE_VERSION', 3) if gitolite_folder: @@ -172,12 +179,14 @@ def _get_gitolite_command(): raise pagure.exceptions.PagureException( 'Non-supported gitolite version "%s"' % gitolite_version ) + _log.debug('Command: %s', cmd) return cmd def generate_gitolite_acls(): """ Generate the gitolite configuration file for all repos """ + _log.info('Refresh gitolite configuration') pagure.lib.git.write_gitolite_acls( pagure.SESSION, pagure.APP.config['GITOLITE_CONFIG']) @@ -200,6 +209,7 @@ def update_git(obj, repo, repofolder): changes commit them and push them back to the original repo. """ + _log.info('Update the git repo: %s for: %s', repo.path, obj) if not repofolder: return @@ -298,6 +308,8 @@ def clean_git(obj, repo, repofolder): if not repofolder: return + _log.info('Update the git repo: %s to remove: %s', repo.path, obj) + # Get the fork repopath = os.path.join(repofolder, repo.path) @@ -800,6 +812,9 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): :arg filestream: the actual content of the file ''' + _log.info( + 'Addinf file: %s to the git repo: %s', + repo.path, werkzeug.secure_filename(filename)) if not ticketfolder: return @@ -913,6 +928,7 @@ def update_file_in_git( :arg user: the user object with its username and email ''' + _log.info('Updating file: %s in the repo: %s', filename, repo.path) # Get the fork repopath = pagure.get_repo_path(repo) @@ -1170,6 +1186,13 @@ def merge_pull_request( session, request, username, request_folder, domerge=True): ''' Merge the specified pull-request. ''' + if domerge: + _log.info( + '%s asked to merge the pull-request: %s', username, request) + else: + _log.info( + '%s asked to diff the pull-request: %s', username, request) + if request.remote: # Get the fork repopath = pagure.get_remote_repo_path( @@ -1185,18 +1208,21 @@ def merge_pull_request( # Clone the original repo into a temp folder newpath = tempfile.mkdtemp(prefix='pagure-pr-merge') + _log.info(' working directory: %s', newpath) new_repo = pygit2.clone_repository(parentpath, newpath) # Update the start and stop commits in the DB, one last time diff_commits = diff_pull_request( session, request, fork_obj, PagureRepo(parentpath), requestfolder=request_folder, with_diff=False) + _log.info(' %s commit to merge', len(diff_commits)) if request.project.settings.get( 'Enforce_signed-off_commits_in_pull-request', False): for commit in diff_commits: if 'signed-off-by' not in commit.message.lower(): shutil.rmtree(newpath) + _log.info(' Missing a required: signed-off-by: Bailing') raise pagure.exceptions.PagureException( 'This repo enforces that all commits are ' 'signed off by their author. ') @@ -1205,6 +1231,7 @@ def merge_pull_request( branch_ref = get_branch_ref(new_repo, request.branch) if not branch_ref: shutil.rmtree(newpath) + _log.info(' Target branch could not be found') raise pagure.exceptions.BranchNotFoundException( 'Branch %s could not be found in the repo %s' % ( request.branch, request.project.fullname @@ -1215,6 +1242,7 @@ def merge_pull_request( branch = get_branch_ref(fork_obj, request.branch_from) if not branch: shutil.rmtree(newpath) + _log.info(' Branch of origin could not be found') raise pagure.exceptions.BranchNotFoundException( 'Branch %s could not be found in the repo %s' % ( request.branch_from, request.project_from.fullname @@ -1227,14 +1255,17 @@ def merge_pull_request( # Add the fork as remote repo reponame = '%s_%s' % (request.user.user, request.uid) + _log.info(' Adding remote: %s pointing to: %s', reponame, repopath) remote = new_repo.create_remote(reponame, repopath) # Fetch the commits remote.fetch() merge = new_repo.merge(repo_commit.oid) + _log.debug(' Merge: %s', merge) if merge is None: mergecode = new_repo.merge_analysis(repo_commit.oid)[0] + _log.debug(' Mergecode: %s', mergecode) refname = '%s:refs/heads/%s' % (branch_ref.name, request.branch) if ( @@ -1244,6 +1275,7 @@ def merge_pull_request( mergecode & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE)): if domerge: + _log.info(' PR up to date, closing it') pagure.lib.close_pull_request( session, request, username, requestfolder=request_folder) @@ -1252,12 +1284,14 @@ def merge_pull_request( session.commit() except SQLAlchemyError as err: # pragma: no cover session.rollback() + _log.exception(' Could not merge the PR in the DB') pagure.APP.logger.exception(err) raise pagure.exceptions.PagureException( 'Could not close this pull-request') raise pagure.exceptions.PagureException( 'Nothing to do, changes were already merged') else: + _log.info(' PR up to date, reporting it') request.merge_status = 'NO_CHANGE' session.commit() shutil.rmtree(newpath) @@ -1270,6 +1304,7 @@ def merge_pull_request( mergecode & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD)): if domerge: + _log.info(' PR merged using fast-forward') head = new_repo.lookup_reference('HEAD').get_object() if not request.project.settings.get('always_merge', False): if merge is not None: @@ -1292,11 +1327,13 @@ def merge_pull_request( tree, [head.hex, repo_commit.oid.hex]) + _log.info(' New head: %s', commit) PagureRepo.push(ori_remote, refname) fork_obj.run_hook( head.hex, commit, 'refs/heads/%s' % request.branch, username) else: + _log.info(' PR merged using fast-forward, reporting it') request.merge_status = 'FFORWARD' session.commit() shutil.rmtree(newpath) @@ -1306,16 +1343,23 @@ def merge_pull_request( tree = None try: tree = new_repo.index.write_tree() - except pygit2.GitError: + except pygit2.GitError as err: + _log.exception( + ' Could not write down the new tree: merge conflicts') + pagure.APP.logger.exception( + ' Could not write down the new tree: merge conflicts') shutil.rmtree(newpath) if domerge: + _log.info(' Merge conflict: Bailing') raise pagure.exceptions.PagureException('Merge conflicts!') else: + _log.info(' Merge conflict, reporting it') request.merge_status = 'CONFLICTS' session.commit() return 'CONFLICTS' if domerge: + _log.info(' Writing down merge commit') head = new_repo.lookup_reference('HEAD').get_object() user_obj = pagure.lib.get_user(session, username) author = pygit2.Signature( @@ -1329,24 +1373,28 @@ def merge_pull_request( tree, [head.hex, repo_commit.oid.hex]) + _log.info(' New head: %s', commit) PagureRepo.push(ori_remote, refname) fork_obj.run_hook( head.hex, commit, 'refs/heads/%s' % request.branch, username) else: + _log.info(' PR can be merged with a merge commit, reporting it') request.merge_status = 'MERGE' session.commit() shutil.rmtree(newpath) return 'MERGE' # Update status + _log.info(' Closing the PR in the DB') pagure.lib.close_pull_request( session, request, username, requestfolder=request_folder, ) try: # Reset the merge_status of all opened PR to refresh their cache + _log.info(' Clear the cached merged status of the other PRs') pagure.lib.reset_status_pull_request(session, request.project) session.commit() except SQLAlchemyError as err: # pragma: no cover diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 5312c86..36ab1b1 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -30,7 +30,7 @@ import pagure.exceptions import pagure.lib import pagure.lib.git import pagure.forms -from pagure import APP, SESSION, LOG, login_required, __get_file_in_tree +from pagure import APP, SESSION, login_required, __get_file_in_tree def _get_parent_repo_path(repo): @@ -562,7 +562,7 @@ def pull_request_drop_comment( flask.flash('Comment removed') except SQLAlchemyError as err: # pragma: no cover SESSION.rollback() - LOG.error(err) + APP.logger.error(err) flask.flash( 'Could not remove the comment: %s' % commentid, 'error') @@ -632,7 +632,7 @@ def pull_request_edit_comment( flask.flash(message) except SQLAlchemyError, err: # pragma: no cover SESSION.rollback() - LOG.error(err) + APP.logger.error(err) if is_js: return 'error' else: diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index e90937b..7f5d493 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -35,7 +35,7 @@ import pagure.exceptions import pagure.lib import pagure.lib.encoding_utils import pagure.forms -from pagure import (APP, SESSION, LOG, __get_file_in_tree, +from pagure import (APP, SESSION, __get_file_in_tree, login_required, authenticated, urlpattern) @@ -136,7 +136,7 @@ def update_issue(repo, issueid, username=None, namespace=None): except SQLAlchemyError as err: # pragma: no cover is_js = False SESSION.rollback() - LOG.error(err) + APP.logger.error(err) if not is_js: flask.flash( 'Could not remove the comment: %s' % commentid, @@ -428,7 +428,7 @@ def edit_tag(repo, tag, username=None, namespace=None): flask.flash(msg) except SQLAlchemyError as err: # pragma: no cover SESSION.rollback() - LOG.error(err) + APP.logger.error(err) flask.flash('Could not edit tag: %s' % tag, 'error') return flask.redirect(flask.url_for( @@ -575,7 +575,7 @@ def remove_tag(repo, username=None, namespace=None): flask.flash(msg) except SQLAlchemyError as err: # pragma: no cover SESSION.rollback() - LOG.error(err) + APP.logger.error(err) flask.flash( 'Could not remove tag: %s' % ','.join(tags), 'error') @@ -1373,7 +1373,7 @@ def view_issue_raw_file( ktc.to_bytes(data)) except pagure.exceptions.PagureException: # We cannot decode the file, so bail but warn the admins - LOG.exception('File could not be decoded') + APP.logger.exception('File could not be decoded') if encoding: mimetype += '; charset={encoding}'.format(encoding=encoding) @@ -1437,7 +1437,7 @@ def edit_comment_issue( flask.flash(message) except SQLAlchemyError, err: # pragma: no cover SESSION.rollback() - LOG.error(err) + APP.logger.error(err) if is_js: return 'error' flask.flash( diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 5449f31..e7c5d1a 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -49,7 +49,7 @@ import pagure.lib.plugins import pagure.forms import pagure import pagure.ui.plugins -from pagure import (APP, SESSION, LOG, __get_file_in_tree, login_required, +from pagure import (APP, SESSION, __get_file_in_tree, login_required, admin_session_timedout) from pagure.lib import encoding_utils @@ -492,7 +492,7 @@ def view_file(repo, identifier, filename, username=None, namespace=None): Image.open(StringIO(content.data)) output_type = 'image' except IOError as err: - LOG.debug( + APP.logger.debug( 'Failed to load image %s, error: %s', filename, err ) output_type = 'binary' @@ -658,7 +658,7 @@ def view_raw_file( encoding = encoding_utils.guess_encoding(ktc.to_bytes(data)) except pagure.exceptions.PagureException: # We cannot decode the file, so bail but warn the admins - LOG.exception('File could not be decoded') + APP.logger.exception('File could not be decoded') if encoding: mimetype += '; charset={encoding}'.format(encoding=encoding) @@ -699,7 +699,7 @@ def view_blame_file(repo, filename, username=None, namespace=None): content = encoding_utils.decode(content.data) except pagure.exceptions.PagureException: # We cannot decode the file, so bail but warn the admins - LOG.exception('File could not be decoded') + APP.logger.exception('File could not be decoded') flask.abort(500, 'File could not be decoded') lexer = TextLexer() diff --git a/tests/__init__.py b/tests/__init__.py index 7267906..e45f43c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -48,9 +48,13 @@ FAITOUT_URL = 'http://faitout.fedorainfracloud.org/' if os.environ.get('FAITOUT_URL'): FAITOUT_URL = os.environ.get('FAITOUT_URL') HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) -LOG = logging.getLogger("pagure") +LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) +PAGLOG = logging.getLogger('pagure') +PAGLOG.setLevel(logging.CRITICAL) +PAGLOG.handlers = [] + LOG.info('BUILD_ID: %s', os.environ.get('BUILD_ID')) if os.environ.get('BUILD_ID')or os.environ.get('FAITOUT_URL'): @@ -67,7 +71,7 @@ if os.environ.get('BUILD_ID')or os.environ.get('FAITOUT_URL'): pass # Remove the log handlers for the tests -pagure.LOG.handlers = [] +pagure.APP.logger.handlers = [] @contextmanager def user_set(APP, user): diff --git a/tests/test_pagure_flask_ui_old_commit.py b/tests/test_pagure_flask_ui_old_commit.py index 4dc6d07..a1c0968 100644 --- a/tests/test_pagure_flask_ui_old_commit.py +++ b/tests/test_pagure_flask_ui_old_commit.py @@ -63,7 +63,6 @@ class PagureFlaskRepoOldUrltests(tests.Modeltests): pagure.APP.config['EMAIL_SEND'] = False pagure.APP.config['OLD_VIEW_COMMIT_ENABLED'] = False - pagure.LOG.handlers = [] def test_view_commit_old(self): """ Test the view_commit_old endpoint. """