From 773396dad999a74cfdc3a63a363fabd88a1f69fa Mon Sep 17 00:00:00 2001 From: Kai Giokas Date: Mon, 26 Feb 2024 18:04:57 -0500 Subject: [PATCH] Added test for the Khmer language It looks like this was an issue with the `_width_table.py` file being out of date in some way. Closes #4235. --- CHANGES.md | 2 + src/black/_width_table.py | 364 +----------------- .../preview_long_strings__east_asian_width.py | 31 ++ 3 files changed, 40 insertions(+), 357 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bcf6eb44fdb..495a28c5a79 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,8 @@ +- Regenerated the `_width_table.py` and added tests for the Khmer language (#4253) + ### Performance diff --git a/src/black/_width_table.py b/src/black/_width_table.py index f3304e48ed0..2c52cdfa998 100644 --- a/src/black/_width_table.py +++ b/src/black/_width_table.py @@ -1,182 +1,10 @@ # Generated by make_width_table.py -# wcwidth 0.2.6 -# Unicode 15.0.0 +# wcwidth 0.2.13 +# Unicode 15.1.0 from typing import Final, List, Tuple WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ - (0, 0, 0), - (1, 31, -1), - (127, 159, -1), - (768, 879, 0), - (1155, 1161, 0), - (1425, 1469, 0), - (1471, 1471, 0), - (1473, 1474, 0), - (1476, 1477, 0), - (1479, 1479, 0), - (1552, 1562, 0), - (1611, 1631, 0), - (1648, 1648, 0), - (1750, 1756, 0), - (1759, 1764, 0), - (1767, 1768, 0), - (1770, 1773, 0), - (1809, 1809, 0), - (1840, 1866, 0), - (1958, 1968, 0), - (2027, 2035, 0), - (2045, 2045, 0), - (2070, 2073, 0), - (2075, 2083, 0), - (2085, 2087, 0), - (2089, 2093, 0), - (2137, 2139, 0), - (2200, 2207, 0), - (2250, 2273, 0), - (2275, 2306, 0), - (2362, 2362, 0), - (2364, 2364, 0), - (2369, 2376, 0), - (2381, 2381, 0), - (2385, 2391, 0), - (2402, 2403, 0), - (2433, 2433, 0), - (2492, 2492, 0), - (2497, 2500, 0), - (2509, 2509, 0), - (2530, 2531, 0), - (2558, 2558, 0), - (2561, 2562, 0), - (2620, 2620, 0), - (2625, 2626, 0), - (2631, 2632, 0), - (2635, 2637, 0), - (2641, 2641, 0), - (2672, 2673, 0), - (2677, 2677, 0), - (2689, 2690, 0), - (2748, 2748, 0), - (2753, 2757, 0), - (2759, 2760, 0), - (2765, 2765, 0), - (2786, 2787, 0), - (2810, 2815, 0), - (2817, 2817, 0), - (2876, 2876, 0), - (2879, 2879, 0), - (2881, 2884, 0), - (2893, 2893, 0), - (2901, 2902, 0), - (2914, 2915, 0), - (2946, 2946, 0), - (3008, 3008, 0), - (3021, 3021, 0), - (3072, 3072, 0), - (3076, 3076, 0), - (3132, 3132, 0), - (3134, 3136, 0), - (3142, 3144, 0), - (3146, 3149, 0), - (3157, 3158, 0), - (3170, 3171, 0), - (3201, 3201, 0), - (3260, 3260, 0), - (3263, 3263, 0), - (3270, 3270, 0), - (3276, 3277, 0), - (3298, 3299, 0), - (3328, 3329, 0), - (3387, 3388, 0), - (3393, 3396, 0), - (3405, 3405, 0), - (3426, 3427, 0), - (3457, 3457, 0), - (3530, 3530, 0), - (3538, 3540, 0), - (3542, 3542, 0), - (3633, 3633, 0), - (3636, 3642, 0), - (3655, 3662, 0), - (3761, 3761, 0), - (3764, 3772, 0), - (3784, 3790, 0), - (3864, 3865, 0), - (3893, 3893, 0), - (3895, 3895, 0), - (3897, 3897, 0), - (3953, 3966, 0), - (3968, 3972, 0), - (3974, 3975, 0), - (3981, 3991, 0), - (3993, 4028, 0), - (4038, 4038, 0), - (4141, 4144, 0), - (4146, 4151, 0), - (4153, 4154, 0), - (4157, 4158, 0), - (4184, 4185, 0), - (4190, 4192, 0), - (4209, 4212, 0), - (4226, 4226, 0), - (4229, 4230, 0), - (4237, 4237, 0), - (4253, 4253, 0), (4352, 4447, 2), - (4957, 4959, 0), - (5906, 5908, 0), - (5938, 5939, 0), - (5970, 5971, 0), - (6002, 6003, 0), - (6068, 6069, 0), - (6071, 6077, 0), - (6086, 6086, 0), - (6089, 6099, 0), - (6109, 6109, 0), - (6155, 6157, 0), - (6159, 6159, 0), - (6277, 6278, 0), - (6313, 6313, 0), - (6432, 6434, 0), - (6439, 6440, 0), - (6450, 6450, 0), - (6457, 6459, 0), - (6679, 6680, 0), - (6683, 6683, 0), - (6742, 6742, 0), - (6744, 6750, 0), - (6752, 6752, 0), - (6754, 6754, 0), - (6757, 6764, 0), - (6771, 6780, 0), - (6783, 6783, 0), - (6832, 6862, 0), - (6912, 6915, 0), - (6964, 6964, 0), - (6966, 6970, 0), - (6972, 6972, 0), - (6978, 6978, 0), - (7019, 7027, 0), - (7040, 7041, 0), - (7074, 7077, 0), - (7080, 7081, 0), - (7083, 7085, 0), - (7142, 7142, 0), - (7144, 7145, 0), - (7149, 7149, 0), - (7151, 7153, 0), - (7212, 7219, 0), - (7222, 7223, 0), - (7376, 7378, 0), - (7380, 7392, 0), - (7394, 7400, 0), - (7405, 7405, 0), - (7412, 7412, 0), - (7416, 7417, 0), - (7616, 7679, 0), - (8203, 8207, 0), - (8232, 8238, 0), - (8288, 8291, 0), - (8400, 8432, 0), (8986, 8987, 2), (9001, 9002, 2), (9193, 9196, 2), @@ -211,183 +39,31 @@ (11035, 11036, 2), (11088, 11088, 2), (11093, 11093, 2), - (11503, 11505, 0), - (11647, 11647, 0), - (11744, 11775, 0), (11904, 11929, 2), (11931, 12019, 2), (12032, 12245, 2), - (12272, 12283, 2), - (12288, 12329, 2), - (12330, 12333, 0), - (12334, 12350, 2), + (12272, 12329, 2), + (12336, 12350, 2), (12353, 12438, 2), - (12441, 12442, 0), (12443, 12543, 2), (12549, 12591, 2), (12593, 12686, 2), (12688, 12771, 2), - (12784, 12830, 2), + (12783, 12830, 2), (12832, 12871, 2), (12880, 19903, 2), (19968, 42124, 2), (42128, 42182, 2), - (42607, 42610, 0), - (42612, 42621, 0), - (42654, 42655, 0), - (42736, 42737, 0), - (43010, 43010, 0), - (43014, 43014, 0), - (43019, 43019, 0), - (43045, 43046, 0), - (43052, 43052, 0), - (43204, 43205, 0), - (43232, 43249, 0), - (43263, 43263, 0), - (43302, 43309, 0), - (43335, 43345, 0), (43360, 43388, 2), - (43392, 43394, 0), - (43443, 43443, 0), - (43446, 43449, 0), - (43452, 43453, 0), - (43493, 43493, 0), - (43561, 43566, 0), - (43569, 43570, 0), - (43573, 43574, 0), - (43587, 43587, 0), - (43596, 43596, 0), - (43644, 43644, 0), - (43696, 43696, 0), - (43698, 43700, 0), - (43703, 43704, 0), - (43710, 43711, 0), - (43713, 43713, 0), - (43756, 43757, 0), - (43766, 43766, 0), - (44005, 44005, 0), - (44008, 44008, 0), - (44013, 44013, 0), (44032, 55203, 2), (63744, 64255, 2), - (64286, 64286, 0), - (65024, 65039, 0), (65040, 65049, 2), - (65056, 65071, 0), (65072, 65106, 2), (65108, 65126, 2), (65128, 65131, 2), (65281, 65376, 2), (65504, 65510, 2), - (66045, 66045, 0), - (66272, 66272, 0), - (66422, 66426, 0), - (68097, 68099, 0), - (68101, 68102, 0), - (68108, 68111, 0), - (68152, 68154, 0), - (68159, 68159, 0), - (68325, 68326, 0), - (68900, 68903, 0), - (69291, 69292, 0), - (69373, 69375, 0), - (69446, 69456, 0), - (69506, 69509, 0), - (69633, 69633, 0), - (69688, 69702, 0), - (69744, 69744, 0), - (69747, 69748, 0), - (69759, 69761, 0), - (69811, 69814, 0), - (69817, 69818, 0), - (69826, 69826, 0), - (69888, 69890, 0), - (69927, 69931, 0), - (69933, 69940, 0), - (70003, 70003, 0), - (70016, 70017, 0), - (70070, 70078, 0), - (70089, 70092, 0), - (70095, 70095, 0), - (70191, 70193, 0), - (70196, 70196, 0), - (70198, 70199, 0), - (70206, 70206, 0), - (70209, 70209, 0), - (70367, 70367, 0), - (70371, 70378, 0), - (70400, 70401, 0), - (70459, 70460, 0), - (70464, 70464, 0), - (70502, 70508, 0), - (70512, 70516, 0), - (70712, 70719, 0), - (70722, 70724, 0), - (70726, 70726, 0), - (70750, 70750, 0), - (70835, 70840, 0), - (70842, 70842, 0), - (70847, 70848, 0), - (70850, 70851, 0), - (71090, 71093, 0), - (71100, 71101, 0), - (71103, 71104, 0), - (71132, 71133, 0), - (71219, 71226, 0), - (71229, 71229, 0), - (71231, 71232, 0), - (71339, 71339, 0), - (71341, 71341, 0), - (71344, 71349, 0), - (71351, 71351, 0), - (71453, 71455, 0), - (71458, 71461, 0), - (71463, 71467, 0), - (71727, 71735, 0), - (71737, 71738, 0), - (71995, 71996, 0), - (71998, 71998, 0), - (72003, 72003, 0), - (72148, 72151, 0), - (72154, 72155, 0), - (72160, 72160, 0), - (72193, 72202, 0), - (72243, 72248, 0), - (72251, 72254, 0), - (72263, 72263, 0), - (72273, 72278, 0), - (72281, 72283, 0), - (72330, 72342, 0), - (72344, 72345, 0), - (72752, 72758, 0), - (72760, 72765, 0), - (72767, 72767, 0), - (72850, 72871, 0), - (72874, 72880, 0), - (72882, 72883, 0), - (72885, 72886, 0), - (73009, 73014, 0), - (73018, 73018, 0), - (73020, 73021, 0), - (73023, 73029, 0), - (73031, 73031, 0), - (73104, 73105, 0), - (73109, 73109, 0), - (73111, 73111, 0), - (73459, 73460, 0), - (73472, 73473, 0), - (73526, 73530, 0), - (73536, 73536, 0), - (73538, 73538, 0), - (78912, 78912, 0), - (78919, 78933, 0), - (92912, 92916, 0), - (92976, 92982, 0), - (94031, 94031, 0), - (94095, 94098, 0), (94176, 94179, 2), - (94180, 94180, 0), - (94192, 94193, 2), (94208, 100343, 2), (100352, 101589, 2), (101632, 101640, 2), @@ -400,32 +76,6 @@ (110933, 110933, 2), (110948, 110951, 2), (110960, 111355, 2), - (113821, 113822, 0), - (118528, 118573, 0), - (118576, 118598, 0), - (119143, 119145, 0), - (119163, 119170, 0), - (119173, 119179, 0), - (119210, 119213, 0), - (119362, 119364, 0), - (121344, 121398, 0), - (121403, 121452, 0), - (121461, 121461, 0), - (121476, 121476, 0), - (121499, 121503, 0), - (121505, 121519, 0), - (122880, 122886, 0), - (122888, 122904, 0), - (122907, 122913, 0), - (122915, 122916, 0), - (122918, 122922, 0), - (123023, 123023, 0), - (123184, 123190, 0), - (123566, 123566, 0), - (123628, 123631, 0), - (124140, 124143, 0), - (125136, 125142, 0), - (125252, 125258, 0), (126980, 126980, 2), (127183, 127183, 2), (127374, 127374, 2), @@ -443,7 +93,8 @@ (127951, 127955, 2), (127968, 127984, 2), (127988, 127988, 2), - (127992, 128062, 2), + (127992, 127994, 2), + (128000, 128062, 2), (128064, 128064, 2), (128066, 128252, 2), (128255, 128317, 2), @@ -474,5 +125,4 @@ (129776, 129784, 2), (131072, 196605, 2), (196608, 262141, 2), - (917760, 917999, 0), ] diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py index 022b0452522..136624cab2e 100644 --- a/tests/data/cases/preview_long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -5,6 +5,20 @@ hangul = '코드포인트 수는 적으나 실제 터미널이나 에디터에서 렌더링될 땐 너무 길어서 줄바꿈이 필요한 문자열' hanzi = '中文測試：代碼點數量少，但在真正的終端模擬器或編輯器中呈現時太長，因此需要換行的字符串。' japanese = 'コードポイントの数は少ないが、実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、改行が要る文字列' +khmer = 'សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ' +# Should stay the same +khmer2 = [ + "text, expected_language", + [ + ( + ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ " + ), + "km", + ), # Khmer + ], +] # output @@ -24,3 +38,20 @@ "実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、" "改行が要る文字列" ) +khmer = ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ" +) +# Should stay the same +khmer2 = [ + "text, expected_language", + [ + ( + ( + "សម្រស់ទាវ២០២២ មិនធម្មតា ឥឡូវកំពុងរកតួ នេនទុំ និងពេជ្រ" + " ប្រញាប់ឡើងទាន់គេមានបញ្ហាត្រូវថតឡើងវិញ " + ), + "km", + ), # Khmer + ], +] \ No newline at end of file