# -*- coding: utf-8 -*-
"""
Tests for :module:`pagure.lib.encoding_utils`.
"""
from __future__ import unicode_literals, absolute_import
import chardet
import os
import unittest
import sys
sys.path.insert(
0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
)
from pagure.lib import encoding_utils
class TestGuessEncoding(unittest.TestCase):
def test_guess_encoding_ascii(self):
"""
Assert when ascii-only data is provided ascii is the guessed encoding.
"""
data = "Twas bryllyg, and the slythy toves did gyre and gymble"
result = encoding_utils.guess_encoding(data.encode("ascii"))
self.assertEqual(result, "ascii")
def test_guess_encoding_favor_utf_8(self):
"""
Test that strings that could be UTF-8 or ISO-8859-* result in UTF-8.
python-chardet-3.0.4-2.fc27.noarch detects it as ISO-8859-9
python-chardet-2.2.1-1.el7_1.noarch detects it as ISO-8859-2
"""
data = "Šabata".encode("utf-8")
result = encoding_utils.guess_encoding(data)
chardet_result = chardet.detect(data)
self.assertEqual(result, "utf-8")
if chardet.__version__[0] == "3":
self.assertEqual(chardet_result["encoding"], "ISO-8859-9")
else:
self.assertEqual(chardet_result["encoding"], "ISO-8859-2")
def test_guess_encoding_no_data(self):
""" Test encoding_utils.guess_encoding() with an empty string """
result = encoding_utils.guess_encoding("".encode("utf-8"))
self.assertEqual(result, "ascii")
class TestGuessEncodings(unittest.TestCase):
def test_guess_encodings(self):
""" Test the encoding_utils.guess_encodings() method. """
data = "Šabata".encode("utf-8")
result = encoding_utils.guess_encodings(data)
chardet_result = chardet.detect(data)
if chardet.__version__[0] == "3":
# The first three have different confidence values
self.assertListEqual(
[encoding.encoding for encoding in result][:3],
["utf-8", "ISO-8859-9", "ISO-8859-1"],
)
# This is the one with the least confidence
self.assertEqual(result[-1].encoding, "windows-1255")
# The values in the middle of the list all have the same confidence
# value and can't be sorted reliably: use sets.
self.assertEqual(
set([encoding.encoding for encoding in result]),
set(
[
"utf-8",
"ISO-8859-9",
"ISO-8859-1",
"MacCyrillic",
"IBM866",
"TIS-620",
"EUC-JP",
"EUC-KR",
"GB2312",
"KOI8-R",
"Big5",
"IBM855",
"ISO-8859-7",
"SHIFT_JIS",
"windows-1253",
"CP949",
"EUC-TW",
"ISO-8859-5",
"windows-1251",
"windows-1255",
]
),
)
self.assertEqual(chardet_result["encoding"], "ISO-8859-9")
else:
self.assertListEqual(
[encoding.encoding for encoding in result],
["utf-8", "ISO-8859-2", "windows-1252"],
)
self.assertEqual(chardet_result["encoding"], "ISO-8859-2")
def test_guess_encodings_no_data(self):
""" Test encoding_utils.guess_encodings() with an emtpy string """
result = encoding_utils.guess_encodings("".encode("utf-8"))
self.assertEqual([encoding.encoding for encoding in result], ["ascii"])
class TestDecode(unittest.TestCase):
def test_decode(self):
""" Test encoding_utils.decode() """
data = "Šabata"
self.assertEqual(data, encoding_utils.decode(data.encode("utf-8")))
if __name__ == "__main__":
unittest.main(verbosity=2)