Blame tests/test_pagure_lib_encoding_utils.py

Jeremy Cline e62ce0
# -*- coding: utf-8 -*-
Jeremy Cline e62ce0
"""
Jeremy Cline e62ce0
Tests for :module:`pagure.lib.encoding_utils`.
Jeremy Cline e62ce0
"""
Jeremy Cline e62ce0
Jeremy Cline e62ce0
import chardet
Pierre-Yves Chibon 52bc45
import os
Jeremy Cline e62ce0
import unittest
Pierre-Yves Chibon 52bc45
import sys
Pierre-Yves Chibon 52bc45
Pierre-Yves Chibon 52bc45
sys.path.insert(0, os.path.join(os.path.dirname(
Pierre-Yves Chibon 52bc45
    os.path.abspath(__file__)), '..'))
Jeremy Cline e62ce0
Jeremy Cline e62ce0
from pagure.lib import encoding_utils
Jeremy Cline e62ce0
Jeremy Cline e62ce0
Jeremy Cline e62ce0
class TestGuessEncoding(unittest.TestCase):
Jeremy Cline e62ce0
Jeremy Cline e62ce0
    def test_guess_encoding_ascii(self):
Jeremy Cline e62ce0
        """
Jeremy Cline e62ce0
        Assert when ascii-only data is provided ascii is the guessed encoding.
Jeremy Cline e62ce0
        """
Jeremy Cline e62ce0
        data = u'Twas bryllyg, and the slythy toves did gyre and gymble'
Jeremy Cline e62ce0
        result = encoding_utils.guess_encoding(data.encode('ascii'))
Jeremy Cline e62ce0
        self.assertEqual(result, 'ascii')
Jeremy Cline e62ce0
Jeremy Cline e62ce0
    def test_guess_encoding_favor_utf_8(self):
Jeremy Cline e62ce0
        """
Pierre-Yves Chibon 8ac14c
        Test that strings that could be UTF-8 or ISO-8859-* result in UTF-8.
Pierre-Yves Chibon 8ac14c
Pierre-Yves Chibon 8ac14c
        python-chardet-3.0.4-2.fc27.noarch detects it as ISO-8859-9
Pierre-Yves Chibon 8ac14c
        python-chardet-2.2.1-1.el7_1.noarch detects it as ISO-8859-2
Jeremy Cline e62ce0
        """
Jeremy Cline e62ce0
        data = u'Šabata'.encode('utf-8')
Jeremy Cline e62ce0
        result = encoding_utils.guess_encoding(data)
Jeremy Cline e62ce0
        chardet_result = chardet.detect(data)
Jeremy Cline e62ce0
        self.assertEqual(result, 'utf-8')
Pierre-Yves Chibon 8ac14c
        if chardet.__version__[0] == '3':
Pierre-Yves Chibon 8ac14c
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
Pierre-Yves Chibon 8ac14c
        else:
Pierre-Yves Chibon 8ac14c
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')
Jeremy Cline e62ce0
Jeremy Cline 4bcf65
    def test_guess_encoding_no_data(self):
Pierre-Yves Chibon 826adb
        """ Test encoding_utils.guess_encoding() with an empty string """
Jeremy Cline 4bcf65
        result = encoding_utils.guess_encoding(u''.encode('utf-8'))
Jeremy Cline 4bcf65
        self.assertEqual(result, 'ascii')
Jeremy Cline 4bcf65
Jeremy Cline 4bcf65
Pierre-Yves Chibon 29890e
class TestGuessEncodings(unittest.TestCase):
Pierre-Yves Chibon 29890e
Pierre-Yves Chibon 29890e
    def test_guess_encodings(self):
Pierre-Yves Chibon 29890e
        """ Test the encoding_utils.guess_encodings() method. """
Pierre-Yves Chibon 29890e
        data = u'Šabata'.encode('utf-8')
Pierre-Yves Chibon 29890e
        result = encoding_utils.guess_encodings(data)
Pierre-Yves Chibon 29890e
        chardet_result = chardet.detect(data)
Pierre-Yves Chibon 8ac14c
        if chardet.__version__[0] == '3':
Pierre-Yves Chibon 8ac14c
            self.assertEqual(
Pierre-Yves Chibon 8ac14c
                [encoding.encoding for encoding in result],
Pierre-Yves Chibon 8ac14c
                ['utf-8', 'ISO-8859-9', 'ISO-8859-1', 'MacCyrillic',
Pierre-Yves Chibon 8ac14c
                 'IBM866', 'TIS-620', 'EUC-JP', 'EUC-KR', 'GB2312', 'KOI8-R',
Pierre-Yves Chibon 8ac14c
                 'Big5', 'IBM855', 'ISO-8859-7', 'SHIFT_JIS', 'windows-1253',
Pierre-Yves Chibon 8ac14c
                 'CP949', 'EUC-TW', 'ISO-8859-5', 'windows-1251',
Pierre-Yves Chibon 8ac14c
                 'windows-1255'])
Pierre-Yves Chibon 8ac14c
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
Pierre-Yves Chibon 8ac14c
        else:
Pierre-Yves Chibon 8ac14c
            self.assertEqual(
Pierre-Yves Chibon 8ac14c
                [encoding.encoding for encoding in result],
Pierre-Yves Chibon 8ac14c
                ['utf-8', 'ISO-8859-2', 'windows-1252'])
Pierre-Yves Chibon 8ac14c
            self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')
Pierre-Yves Chibon 29890e
Pierre-Yves Chibon 29890e
    def test_guess_encodings_no_data(self):
Pierre-Yves Chibon 29890e
        """ Test encoding_utils.guess_encodings() with an emtpy string """
Pierre-Yves Chibon 29890e
        result = encoding_utils.guess_encodings(u''.encode('utf-8'))
Pierre-Yves Chibon 29890e
        self.assertEqual(
Pierre-Yves Chibon 29890e
            [encoding.encoding for encoding in result],
Pierre-Yves Chibon 29890e
            ['ascii'])
Pierre-Yves Chibon 29890e
Jeremy Cline 4bcf65
class TestDecode(unittest.TestCase):
Jeremy Cline 4bcf65
Jeremy Cline e62ce0
    def test_decode(self):
Pierre-Yves Chibon 52bc45
        """ Test encoding_utils.decode() """
Jeremy Cline e62ce0
        data = u'Šabata'
Jeremy Cline e62ce0
        self.assertEqual(data, encoding_utils.decode(data.encode('utf-8')))
Jeremy Cline e62ce0
Jeremy Cline e62ce0
Jeremy Cline e62ce0
if __name__ == '__main__':
Pierre-Yves Chibon 52bc45
    unittest.main(verbosity=2)