import unittest from . import utils class TestValueStringSingle(unittest.TestCase): def test_empty(self): result = utils.list_tokens(" '' ") self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, '') result = utils.list_tokens(" '''''' ") self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, '') def test_content(self): result = utils.list_tokens(" 'abc def\\t1234' ") self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, 'abc def\t1234') result = utils.list_tokens(" '''abc def\\t1234''' ") self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, 'abc def\t1234') def test_no_newline(self): with self.assertRaises(utils.LexError): utils.list_tokens(" 'a\nb' ") result = utils.list_tokens(" '''a\nb''' ") self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, 'a\nb') def test_valid_octals(self): for i in range(256): # Prepare number = oct(i)[2:] number = "'\\%s%s'" % ("0" * (3 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def test_valid_octals_multiline(self): for i in range(256): # Prepare number = oct(i)[2:] number = "'''\\%s%s'''" % ("0" * (3 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def test_valid_hex_lower(self): # Lower x for i in range(0o1000): # Prepare number = hex(i)[2:].lower() number = "'\\x%s%s'" % ("0" * (2 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper x for i in range(0o1000): # Prepare number = hex(i)[2:].lower() number = "'\\X%s%s'" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def test_valid_hex_lower_multiline(self): # Lower x for i in range(0o1000): # Prepare number = hex(i)[2:].lower() number = "'''\\x%s%s'''" % ("0" * (2 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper x for i in range(0o1000): # Prepare number = hex(i)[2:].lower() number = "'''\\X%s%s'''" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def test_valid_hex_upper(self): # Lower x for i in range(256): # Prepare number = hex(i)[2:].upper() number = "'\\x%s%s'" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper x for i in range(256): # Prepare number = hex(i)[2:].upper() number = "'\\X%s%s'" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def test_valid_hex_upper_multiline(self): # Lower x for i in range(256): # Prepare number = hex(i)[2:].upper() number = "'''\\x%s%s'''" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper x for i in range(256): # Prepare number = hex(i)[2:].upper() number = "'''\\X%s%s'''" % ("0" * (2 - len(number)), number) comparison = eval(number.lower()) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) def __filter_codes(self, source): def filter_func(value): if 0xd800 <= value < 0xe000: return False else: return True return filter(filter_func, source) @utils.big_test def test_valid_small_unicode(self): # Upper x for i in self.__filter_codes(range(2 ** 16)): # Prepare number = hex(i)[2:].lower() number = "'\\u%s%s'" % ("0" * (4 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper u for i in self.__filter_codes(range(2 ** 16)): # Prepare number = hex(i)[2:].upper() number = "'\\u%s%s'" % ("0" * (4 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) @utils.big_test def test_valid_small_unicode_multiline(self): # Upper x for i in self.__filter_codes(range(2 ** 16)): # Prepare number = hex(i)[2:].lower() number = "'''\\u%s%s'''" % ("0" * (4 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper u for i in self.__filter_codes(range(2 ** 16)): # Prepare number = hex(i)[2:].upper() number = "'''\\u%s%s'''" % ("0" * (4 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) @utils.big_test def test_valid_big_unicode(self): # Upper x for i in self.__filter_codes(range(0x00110000)): # Prepare number = hex(i)[2:].lower() number = "'\\U%s%s'" % ("0" * (8 - len(number)), number) try: comparison = eval(number) # TODO: Better string generation except: raise ValueError(number) # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper u for i in self.__filter_codes(range(0x00110000)): # Prepare number = hex(i)[2:].upper() number = "'\\U%s%s'" % ("0" * (8 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) @utils.big_test def test_valid_big_unicode_multiline(self): # Upper x for i in self.__filter_codes(range(0x00110000)): # Prepare number = hex(i)[2:].lower() number = "'''\\U%s%s'''" % ("0" * (8 - len(number)), number) try: comparison = eval(number) # TODO: Better string generation except: raise ValueError(number) # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) # Upper u for i in self.__filter_codes(range(0x00110000)): # Prepare number = hex(i)[2:].upper() number = "'''\\U%s%s'''" % ("0" * (8 - len(number)), number) comparison = eval(number) # TODO: Better string generation # Test result = utils.list_tokens(number) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, comparison) _special_chars = {"\\": "\\", "'": "'", "a": "\x07", "b": "\x08", "f": "\x0c", "n": "\n", "r": "\r", "t": "\t", "v": "\v", "000": "\000", "100": "\100", "200": "\200", "300": "\300", "400": "\400", "500": "\500", "600": "\600", "700": "\700", "x00": "\x00", "X00": "\x00", "u0000": "\x00", "U00000000": "\x00"} def test_special_values(self): for code, code_result in self._special_chars.items(): result = utils.list_tokens(" '\\%s' " % code) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, code_result) result = utils.list_tokens(" '''\\%s''' " % code) self.assertEqual(len(result), 1) self.assertEqual(result[0].type, "VALUE_STRING") self.assertEqual(result[0].value, code_result) def test_not_special_values(self): ignore = set(map(lambda x: x[0], self._special_chars)) all_chars = set(map(lambda x: bytes([x]).decode(), range(128))) tmp = all_chars - ignore for code in tmp: with self.assertRaises(utils.LexError): utils.list_tokens(" '\\%s' " % (code,)) with self.assertRaises(utils.LexError): utils.list_tokens(" '''\\%s''' " % code)