339 lines
13 KiB
Python
339 lines
13 KiB
Python
import unittest
|
|
from . import utils
|
|
|
|
|
|
class TestValueStringSingle(unittest.TestCase):
|
|
def test_empty(self):
|
|
result = utils.list_tokens(" '' ")
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, '')
|
|
|
|
result = utils.list_tokens(" '''''' ")
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, '')
|
|
|
|
def test_content(self):
|
|
result = utils.list_tokens(" 'abc def\\t1234' ")
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, 'abc def\t1234')
|
|
|
|
result = utils.list_tokens(" '''abc def\\t1234''' ")
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, 'abc def\t1234')
|
|
|
|
def test_no_newline(self):
|
|
with self.assertRaises(utils.LexError):
|
|
utils.list_tokens(" 'a\nb' ")
|
|
|
|
result = utils.list_tokens(" '''a\nb''' ")
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, 'a\nb')
|
|
|
|
def test_valid_octals(self):
|
|
for i in range(256):
|
|
# Prepare
|
|
number = oct(i)[2:]
|
|
number = "'\\%s%s'" % ("0" * (3 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def test_valid_octals_multiline(self):
|
|
for i in range(256):
|
|
# Prepare
|
|
number = oct(i)[2:]
|
|
number = "'''\\%s%s'''" % ("0" * (3 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def test_valid_hex_lower(self):
|
|
# Lower x
|
|
for i in range(0o1000):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'\\x%s%s'" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper x
|
|
for i in range(0o1000):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'\\X%s%s'" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def test_valid_hex_lower_multiline(self):
|
|
# Lower x
|
|
for i in range(0o1000):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'''\\x%s%s'''" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper x
|
|
for i in range(0o1000):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'''\\X%s%s'''" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def test_valid_hex_upper(self):
|
|
# Lower x
|
|
for i in range(256):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'\\x%s%s'" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper x
|
|
for i in range(256):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'\\X%s%s'" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def test_valid_hex_upper_multiline(self):
|
|
# Lower x
|
|
for i in range(256):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'''\\x%s%s'''" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper x
|
|
for i in range(256):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'''\\X%s%s'''" % ("0" * (2 - len(number)), number)
|
|
comparison = eval(number.lower()) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
def __filter_codes(self, source):
|
|
def filter_func(value):
|
|
if 0xd800 <= value < 0xe000:
|
|
return False
|
|
else:
|
|
return True
|
|
return filter(filter_func, source)
|
|
|
|
@utils.big_test
|
|
def test_valid_small_unicode(self):
|
|
# Upper x
|
|
for i in self.__filter_codes(range(2 ** 16)):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'\\u%s%s'" % ("0" * (4 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper u
|
|
for i in self.__filter_codes(range(2 ** 16)):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'\\u%s%s'" % ("0" * (4 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
@utils.big_test
|
|
def test_valid_small_unicode_multiline(self):
|
|
# Upper x
|
|
for i in self.__filter_codes(range(2 ** 16)):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'''\\u%s%s'''" % ("0" * (4 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper u
|
|
for i in self.__filter_codes(range(2 ** 16)):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'''\\u%s%s'''" % ("0" * (4 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
@utils.big_test
|
|
def test_valid_big_unicode(self):
|
|
# Upper x
|
|
for i in self.__filter_codes(range(0x00110000)):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'\\U%s%s'" % ("0" * (8 - len(number)), number)
|
|
try:
|
|
comparison = eval(number) # TODO: Better string generation
|
|
except:
|
|
raise ValueError(number)
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper u
|
|
for i in self.__filter_codes(range(0x00110000)):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'\\U%s%s'" % ("0" * (8 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
@utils.big_test
|
|
def test_valid_big_unicode_multiline(self):
|
|
# Upper x
|
|
for i in self.__filter_codes(range(0x00110000)):
|
|
# Prepare
|
|
number = hex(i)[2:].lower()
|
|
number = "'''\\U%s%s'''" % ("0" * (8 - len(number)), number)
|
|
try:
|
|
comparison = eval(number) # TODO: Better string generation
|
|
except:
|
|
raise ValueError(number)
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
# Upper u
|
|
for i in self.__filter_codes(range(0x00110000)):
|
|
# Prepare
|
|
number = hex(i)[2:].upper()
|
|
number = "'''\\U%s%s'''" % ("0" * (8 - len(number)), number)
|
|
comparison = eval(number) # TODO: Better string generation
|
|
|
|
# Test
|
|
result = utils.list_tokens(number)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, comparison)
|
|
|
|
_special_chars = {"\\": "\\",
|
|
"'": "'",
|
|
"a": "\x07",
|
|
"b": "\x08",
|
|
"f": "\x0c",
|
|
"n": "\n",
|
|
"r": "\r",
|
|
"t": "\t",
|
|
"v": "\v",
|
|
"000": "\000",
|
|
"100": "\100",
|
|
"200": "\200",
|
|
"300": "\300",
|
|
"400": "\400",
|
|
"500": "\500",
|
|
"600": "\600",
|
|
"700": "\700",
|
|
"x00": "\x00",
|
|
"X00": "\x00",
|
|
"u0000": "\x00",
|
|
"U00000000": "\x00"}
|
|
|
|
def test_special_values(self):
|
|
for code, code_result in self._special_chars.items():
|
|
result = utils.list_tokens(" '\\%s' " % code)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, code_result)
|
|
|
|
result = utils.list_tokens(" '''\\%s''' " % code)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].type, "VALUE_STRING")
|
|
self.assertEqual(result[0].value, code_result)
|
|
|
|
def test_not_special_values(self):
|
|
ignore = set(map(lambda x: x[0], self._special_chars))
|
|
all_chars = set(map(lambda x: bytes([x]).decode(), range(128)))
|
|
tmp = all_chars - ignore
|
|
for code in tmp:
|
|
with self.assertRaises(utils.LexError):
|
|
utils.list_tokens(" '\\%s' " % (code,))
|
|
with self.assertRaises(utils.LexError):
|
|
utils.list_tokens(" '''\\%s''' " % code) |