UNICODE_ARRAY and ASCII_ARRAY fixes.

- Avoid unnecessary escaping of single quotes within string literals.
- Add a missing blank line between two `UNICODE_ARRAY` sections.
This commit is contained in:
Nicholas Nethercote 2024-05-29 16:20:19 +10:00
parent a330e49593
commit e1ae0fa055

View File

@ -129,42 +129,42 @@
('。', "Ideographic Full Stop", "."),
('︒', "Presentation Form For Vertical Ideographic Full Stop", "."),
('՝', "Armenian Comma", "\'"),
('', "Fullwidth Apostrophe", "\'"),
('', "Left Single Quotation Mark", "\'"),
('', "Right Single Quotation Mark", "\'"),
('', "Single High-Reversed-9 Quotation Mark", "\'"),
('', "Prime", "\'"),
('', "Reversed Prime", "\'"),
('՚', "Armenian Apostrophe", "\'"),
('׳', "Hebrew Punctuation Geresh", "\'"),
('`', "Grave Accent", "\'"),
('', "Greek Varia", "\'"),
('', "Fullwidth Grave Accent", "\'"),
('´', "Acute Accent", "\'"),
('΄', "Greek Tonos", "\'"),
('', "Greek Oxia", "\'"),
('', "Greek Koronis", "\'"),
('᾿', "Greek Psili", "\'"),
('', "Greek Dasia", "\'"),
('ʹ', "Modifier Letter Prime", "\'"),
('ʹ', "Greek Numeral Sign", "\'"),
('ˈ', "Modifier Letter Vertical Line", "\'"),
('ˊ', "Modifier Letter Acute Accent", "\'"),
('ˋ', "Modifier Letter Grave Accent", "\'"),
('˴', "Modifier Letter Middle Grave Accent", "\'"),
('ʻ', "Modifier Letter Turned Comma", "\'"),
('ʽ', "Modifier Letter Reversed Comma", "\'"),
('ʼ', "Modifier Letter Apostrophe", "\'"),
('ʾ', "Modifier Letter Right Half Ring", "\'"),
('', "Latin Small Letter Saltillo", "\'"),
('י', "Hebrew Letter Yod", "\'"),
('ߴ', "Nko High Tone Apostrophe", "\'"),
('ߵ', "Nko Low Tone Apostrophe", "\'"),
('', "Canadian Syllabics West-Cree P", "\'"),
('', "Runic Letter Short-Twig-Sol S", "\'"),
('𖽑', "Miao Sign Aspiration", "\'"),
('𖽒', "Miao Sign Reformed Voicing", "\'"),
('՝', "Armenian Comma", "'"),
('', "Fullwidth Apostrophe", "'"),
('', "Left Single Quotation Mark", "'"),
('', "Right Single Quotation Mark", "'"),
('', "Single High-Reversed-9 Quotation Mark", "'"),
('', "Prime", "'"),
('', "Reversed Prime", "'"),
('՚', "Armenian Apostrophe", "'"),
('׳', "Hebrew Punctuation Geresh", "'"),
('`', "Grave Accent", "'"),
('', "Greek Varia", "'"),
('', "Fullwidth Grave Accent", "'"),
('´', "Acute Accent", "'"),
('΄', "Greek Tonos", "'"),
('', "Greek Oxia", "'"),
('', "Greek Koronis", "'"),
('᾿', "Greek Psili", "'"),
('', "Greek Dasia", "'"),
('ʹ', "Modifier Letter Prime", "'"),
('ʹ', "Greek Numeral Sign", "'"),
('ˈ', "Modifier Letter Vertical Line", "'"),
('ˊ', "Modifier Letter Acute Accent", "'"),
('ˋ', "Modifier Letter Grave Accent", "'"),
('˴', "Modifier Letter Middle Grave Accent", "'"),
('ʻ', "Modifier Letter Turned Comma", "'"),
('ʽ', "Modifier Letter Reversed Comma", "'"),
('ʼ', "Modifier Letter Apostrophe", "'"),
('ʾ', "Modifier Letter Right Half Ring", "'"),
('', "Latin Small Letter Saltillo", "'"),
('י', "Hebrew Letter Yod", "'"),
('ߴ', "Nko High Tone Apostrophe", "'"),
('ߵ', "Nko Low Tone Apostrophe", "'"),
('', "Canadian Syllabics West-Cree P", "'"),
('', "Runic Letter Short-Twig-Sol S", "'"),
('𖽑', "Miao Sign Aspiration", "'"),
('𖽒', "Miao Sign Reformed Voicing", "'"),
('᳓', "Vedic Sign Nihshvasa", "\""),
('', "Fullwidth Quotation Mark", "\""),
@ -298,6 +298,7 @@
('〉', "Right Angle Bracket", ">"),
('》', "Right Double Angle Bracket", ">"),
('', "Fullwidth Greater-Than Sign", ">"),
('⩵', "Two Consecutive Equals Signs", "==")
];
@ -332,7 +333,7 @@
(">", "Greater-Than Sign", Some(token::Gt)),
// FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
// spitting the correct token out.
("\'", "Single Quote", None),
("'", "Single Quote", None),
("\"", "Quotation Mark", None),
];