|
Jeremy Cline |
e62ce0 |
# -*- coding: utf-8 -*-
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Jeremy Cline |
e62ce0 |
Tests for :module:`pagure.lib.encoding_utils`.
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
import chardet
|
|
Pierre-Yves Chibon |
52bc45 |
import os
|
|
Jeremy Cline |
e62ce0 |
import unittest
|
|
Pierre-Yves Chibon |
52bc45 |
import sys
|
|
Pierre-Yves Chibon |
52bc45 |
|
|
Pierre-Yves Chibon |
52bc45 |
sys.path.insert(0, os.path.join(os.path.dirname(
|
|
Pierre-Yves Chibon |
52bc45 |
os.path.abspath(__file__)), '..'))
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
from pagure.lib import encoding_utils
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
class TestGuessEncoding(unittest.TestCase):
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
def test_guess_encoding_ascii(self):
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Jeremy Cline |
e62ce0 |
Assert when ascii-only data is provided ascii is the guessed encoding.
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Jeremy Cline |
e62ce0 |
data = u'Twas bryllyg, and the slythy toves did gyre and gymble'
|
|
Jeremy Cline |
e62ce0 |
result = encoding_utils.guess_encoding(data.encode('ascii'))
|
|
Jeremy Cline |
e62ce0 |
self.assertEqual(result, 'ascii')
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
def test_guess_encoding_favor_utf_8(self):
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Pierre-Yves Chibon |
8ac14c |
Test that strings that could be UTF-8 or ISO-8859-* result in UTF-8.
|
|
Pierre-Yves Chibon |
8ac14c |
|
|
Pierre-Yves Chibon |
8ac14c |
python-chardet-3.0.4-2.fc27.noarch detects it as ISO-8859-9
|
|
Pierre-Yves Chibon |
8ac14c |
python-chardet-2.2.1-1.el7_1.noarch detects it as ISO-8859-2
|
|
Jeremy Cline |
e62ce0 |
"""
|
|
Jeremy Cline |
e62ce0 |
data = u'Šabata'.encode('utf-8')
|
|
Jeremy Cline |
e62ce0 |
result = encoding_utils.guess_encoding(data)
|
|
Jeremy Cline |
e62ce0 |
chardet_result = chardet.detect(data)
|
|
Jeremy Cline |
e62ce0 |
self.assertEqual(result, 'utf-8')
|
|
Pierre-Yves Chibon |
8ac14c |
if chardet.__version__[0] == '3':
|
|
Pierre-Yves Chibon |
8ac14c |
self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
|
|
Pierre-Yves Chibon |
8ac14c |
else:
|
|
Pierre-Yves Chibon |
8ac14c |
self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
4bcf65 |
def test_guess_encoding_no_data(self):
|
|
Pierre-Yves Chibon |
826adb |
""" Test encoding_utils.guess_encoding() with an empty string """
|
|
Jeremy Cline |
4bcf65 |
result = encoding_utils.guess_encoding(u''.encode('utf-8'))
|
|
Jeremy Cline |
4bcf65 |
self.assertEqual(result, 'ascii')
|
|
Jeremy Cline |
4bcf65 |
|
|
Jeremy Cline |
4bcf65 |
|
|
Pierre-Yves Chibon |
29890e |
class TestGuessEncodings(unittest.TestCase):
|
|
Pierre-Yves Chibon |
29890e |
|
|
Pierre-Yves Chibon |
29890e |
def test_guess_encodings(self):
|
|
Pierre-Yves Chibon |
29890e |
""" Test the encoding_utils.guess_encodings() method. """
|
|
Pierre-Yves Chibon |
29890e |
data = u'Šabata'.encode('utf-8')
|
|
Pierre-Yves Chibon |
29890e |
result = encoding_utils.guess_encodings(data)
|
|
Pierre-Yves Chibon |
29890e |
chardet_result = chardet.detect(data)
|
|
Pierre-Yves Chibon |
8ac14c |
if chardet.__version__[0] == '3':
|
|
Aurélien Bompard |
f61bb3 |
# The first three have different confidence values
|
|
Aurélien Bompard |
f61bb3 |
self.assertListEqual(
|
|
Aurélien Bompard |
f61bb3 |
[encoding.encoding for encoding in result][:3],
|
|
Aurélien Bompard |
f61bb3 |
['utf-8', 'ISO-8859-9', 'ISO-8859-1']
|
|
Aurélien Bompard |
f61bb3 |
)
|
|
Aurélien Bompard |
f61bb3 |
# This is the one with the least confidence
|
|
Aurélien Bompard |
f61bb3 |
self.assertEqual(result[-1].encoding, 'windows-1255')
|
|
Aurélien Bompard |
f61bb3 |
# The values in the middle of the list all have the same confidence
|
|
Aurélien Bompard |
f61bb3 |
# value and can't be sorted reliably: use sets.
|
|
Pierre-Yves Chibon |
8ac14c |
self.assertEqual(
|
|
Aurélien Bompard |
f61bb3 |
set([encoding.encoding for encoding in result]),
|
|
Aurélien Bompard |
f61bb3 |
set(['utf-8', 'ISO-8859-9', 'ISO-8859-1', 'MacCyrillic',
|
|
Aurélien Bompard |
f61bb3 |
'IBM866', 'TIS-620', 'EUC-JP', 'EUC-KR', 'GB2312',
|
|
Aurélien Bompard |
f61bb3 |
'KOI8-R', 'Big5', 'IBM855', 'ISO-8859-7', 'SHIFT_JIS',
|
|
Aurélien Bompard |
f61bb3 |
'windows-1253', 'CP949', 'EUC-TW', 'ISO-8859-5',
|
|
Aurélien Bompard |
f61bb3 |
'windows-1251', 'windows-1255'])
|
|
Aurélien Bompard |
f61bb3 |
)
|
|
Pierre-Yves Chibon |
8ac14c |
self.assertEqual(chardet_result['encoding'], 'ISO-8859-9')
|
|
Pierre-Yves Chibon |
8ac14c |
else:
|
|
Aurélien Bompard |
f61bb3 |
self.assertListEqual(
|
|
Pierre-Yves Chibon |
8ac14c |
[encoding.encoding for encoding in result],
|
|
Pierre-Yves Chibon |
8ac14c |
['utf-8', 'ISO-8859-2', 'windows-1252'])
|
|
Pierre-Yves Chibon |
8ac14c |
self.assertEqual(chardet_result['encoding'], 'ISO-8859-2')
|
|
Pierre-Yves Chibon |
29890e |
|
|
Pierre-Yves Chibon |
29890e |
def test_guess_encodings_no_data(self):
|
|
Pierre-Yves Chibon |
29890e |
""" Test encoding_utils.guess_encodings() with an emtpy string """
|
|
Pierre-Yves Chibon |
29890e |
result = encoding_utils.guess_encodings(u''.encode('utf-8'))
|
|
Pierre-Yves Chibon |
29890e |
self.assertEqual(
|
|
Pierre-Yves Chibon |
29890e |
[encoding.encoding for encoding in result],
|
|
Pierre-Yves Chibon |
29890e |
['ascii'])
|
|
Pierre-Yves Chibon |
29890e |
|
|
Jeremy Cline |
4bcf65 |
class TestDecode(unittest.TestCase):
|
|
Jeremy Cline |
4bcf65 |
|
|
Jeremy Cline |
e62ce0 |
def test_decode(self):
|
|
Pierre-Yves Chibon |
52bc45 |
""" Test encoding_utils.decode() """
|
|
Jeremy Cline |
e62ce0 |
data = u'Šabata'
|
|
Jeremy Cline |
e62ce0 |
self.assertEqual(data, encoding_utils.decode(data.encode('utf-8')))
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
|
|
Jeremy Cline |
e62ce0 |
if __name__ == '__main__':
|
|
Pierre-Yves Chibon |
52bc45 |
unittest.main(verbosity=2)
|