Blame tests/test_pagure_lib_encoding_utils.py

Jeremy Cline e62ce0
# -*- coding: utf-8 -*-
Jeremy Cline e62ce0
"""
Jeremy Cline e62ce0
Tests for :module:`pagure.lib.encoding_utils`.
Jeremy Cline e62ce0
"""
Jeremy Cline e62ce0
Pierre-Yves Chibon 67d1cc
from __future__ import unicode_literals, absolute_import
Aurélien Bompard 626417
Jeremy Cline e62ce0
import chardet
Pierre-Yves Chibon 52bc45
import os
Jeremy Cline e62ce0
import unittest
Pierre-Yves Chibon 52bc45
import sys
Pierre-Yves Chibon 52bc45
Pierre-Yves Chibon 73d120
sys.path.insert(
Pierre-Yves Chibon 73d120
    0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
Pierre-Yves Chibon 73d120
)
Jeremy Cline e62ce0
Jeremy Cline e62ce0
from pagure.lib import encoding_utils
Jeremy Cline e62ce0
Jeremy Cline e62ce0
Jeremy Cline e62ce0
class TestGuessEncoding(unittest.TestCase):
Jeremy Cline e62ce0
    def test_guess_encoding_ascii(self):
Jeremy Cline e62ce0
        """
Jeremy Cline e62ce0
        Assert when ascii-only data is provided ascii is the guessed encoding.
Jeremy Cline e62ce0
        """
Pierre-Yves Chibon 73d120
        data = "Twas bryllyg, and the slythy toves did gyre and gymble"
Pierre-Yves Chibon 73d120
        result = encoding_utils.guess_encoding(data.encode("ascii"))
Pierre-Yves Chibon 73d120
        self.assertEqual(result, "ascii")
Jeremy Cline e62ce0
Jeremy Cline e62ce0
    def test_guess_encoding_favor_utf_8(self):
Jeremy Cline e62ce0
        """
Pierre-Yves Chibon 8ac14c
        Test that strings that could be UTF-8 or ISO-8859-* result in UTF-8.
Pierre-Yves Chibon 8ac14c
Pierre-Yves Chibon 8ac14c
        python-chardet-3.0.4-2.fc27.noarch detects it as ISO-8859-9
Pierre-Yves Chibon 8ac14c
        python-chardet-2.2.1-1.el7_1.noarch detects it as ISO-8859-2
Jeremy Cline e62ce0
        """
Pierre-Yves Chibon 73d120
        data = "Šabata".encode("utf-8")
Jeremy Cline e62ce0
        result = encoding_utils.guess_encoding(data)
Jeremy Cline e62ce0
        chardet_result = chardet.detect(data)
Pierre-Yves Chibon 73d120
        self.assertEqual(result, "utf-8")
Pierre-Yves Chibon 73d120
        if chardet.__version__[0] == "3":
Pierre-Yves Chibon 73d120
            self.assertEqual(chardet_result["encoding"], "ISO-8859-9")
Pierre-Yves Chibon 8ac14c
        else:
Pierre-Yves Chibon 73d120
            self.assertEqual(chardet_result["encoding"], "ISO-8859-2")
Jeremy Cline e62ce0
Jeremy Cline 4bcf65
    def test_guess_encoding_no_data(self):
Pierre-Yves Chibon 826adb
        """ Test encoding_utils.guess_encoding() with an empty string """
Pierre-Yves Chibon 73d120
        result = encoding_utils.guess_encoding("".encode("utf-8"))
Pierre-Yves Chibon 73d120
        self.assertEqual(result, "ascii")
Jeremy Cline 4bcf65
Jeremy Cline 4bcf65
Pierre-Yves Chibon 29890e
class TestGuessEncodings(unittest.TestCase):
Pierre-Yves Chibon 29890e
    def test_guess_encodings(self):
Pierre-Yves Chibon 29890e
        """ Test the encoding_utils.guess_encodings() method. """
Pierre-Yves Chibon 73d120
        data = "Šabata".encode("utf-8")
Pierre-Yves Chibon 29890e
        result = encoding_utils.guess_encodings(data)
Pierre-Yves Chibon 29890e
        chardet_result = chardet.detect(data)
Pierre-Yves Chibon 73d120
        if chardet.__version__[0] == "3":
Aurélien Bompard f61bb3
            # The first three have different confidence values
Aurélien Bompard f61bb3
            self.assertListEqual(
Aurélien Bompard f61bb3
                [encoding.encoding for encoding in result][:3],
Pierre-Yves Chibon 73d120
                ["utf-8", "ISO-8859-9", "ISO-8859-1"],
Aurélien Bompard f61bb3
            )
Aurélien Bompard f61bb3
            # This is the one with the least confidence
Pierre-Yves Chibon 73d120
            self.assertEqual(result[-1].encoding, "windows-1255")
Aurélien Bompard f61bb3
            # The values in the middle of the list all have the same confidence
Aurélien Bompard f61bb3
            # value and can't be sorted reliably: use sets.
Pierre-Yves Chibon 8ac14c
            self.assertEqual(
Aurélien Bompard f61bb3
                set([encoding.encoding for encoding in result]),
Pierre-Yves Chibon 73d120
                set(
Pierre-Yves Chibon 73d120
                    [
Pierre-Yves Chibon 73d120
                        "utf-8",
Pierre-Yves Chibon 73d120
                        "ISO-8859-9",
Pierre-Yves Chibon 73d120
                        "ISO-8859-1",
Pierre-Yves Chibon 73d120
                        "MacCyrillic",
Pierre-Yves Chibon 73d120
                        "IBM866",
Pierre-Yves Chibon 73d120
                        "TIS-620",
Pierre-Yves Chibon 73d120
                        "EUC-JP",
Pierre-Yves Chibon 73d120
                        "EUC-KR",
Pierre-Yves Chibon 73d120
                        "GB2312",
Pierre-Yves Chibon 73d120
                        "KOI8-R",
Pierre-Yves Chibon 73d120
                        "Big5",
Pierre-Yves Chibon 73d120
                        "IBM855",
Pierre-Yves Chibon 73d120
                        "ISO-8859-7",
Pierre-Yves Chibon 73d120
                        "SHIFT_JIS",
Pierre-Yves Chibon 73d120
                        "windows-1253",
Pierre-Yves Chibon 73d120
                        "CP949",
Pierre-Yves Chibon 73d120
                        "EUC-TW",
Pierre-Yves Chibon 73d120
                        "ISO-8859-5",
Pierre-Yves Chibon 73d120
                        "windows-1251",
Pierre-Yves Chibon 73d120
                        "windows-1255",
Pierre-Yves Chibon 73d120
                    ]
Pierre-Yves Chibon 73d120
                ),
Aurélien Bompard f61bb3
            )
Pierre-Yves Chibon 73d120
            self.assertEqual(chardet_result["encoding"], "ISO-8859-9")
Pierre-Yves Chibon 8ac14c
        else:
Aurélien Bompard f61bb3
            self.assertListEqual(
Pierre-Yves Chibon 8ac14c
                [encoding.encoding for encoding in result],
Pierre-Yves Chibon 73d120
                ["utf-8", "ISO-8859-2", "windows-1252"],
Pierre-Yves Chibon 73d120
            )
Pierre-Yves Chibon 73d120
            self.assertEqual(chardet_result["encoding"], "ISO-8859-2")
Pierre-Yves Chibon 29890e
Pierre-Yves Chibon 29890e
    def test_guess_encodings_no_data(self):
Pierre-Yves Chibon 29890e
        """ Test encoding_utils.guess_encodings() with an emtpy string """
Pierre-Yves Chibon 73d120
        result = encoding_utils.guess_encodings("".encode("utf-8"))
Pierre-Yves Chibon 73d120
        self.assertEqual([encoding.encoding for encoding in result], ["ascii"])
Pierre-Yves Chibon 29890e
Jeremy Cline 4bcf65
Pierre-Yves Chibon 73d120
class TestDecode(unittest.TestCase):
Jeremy Cline e62ce0
    def test_decode(self):
Pierre-Yves Chibon 52bc45
        """ Test encoding_utils.decode() """
Pierre-Yves Chibon 73d120
        data = "Šabata"
Pierre-Yves Chibon 73d120
        self.assertEqual(data, encoding_utils.decode(data.encode("utf-8")))
Jeremy Cline e62ce0
Jeremy Cline e62ce0
Pierre-Yves Chibon 73d120
if __name__ == "__main__":
Pierre-Yves Chibon 52bc45
    unittest.main(verbosity=2)