Clarify and correct comment about multi_byte_tokens

This commit is contained in:
DJMcNab 2018-12-08 14:46:15 +00:00 committed by Aleksey Kladov
parent 3d3026dc60
commit 89cf7d8743

@ -25,9 +25,9 @@ Grammar(
["^", "CARET"],
["%", "PERCENT"],
],
// TODO: Confirm surmision: the tokens which cannot be recorded in a single UTF-8 byte
// Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT)
multi_byte_tokens: [
[".", "DOT"], // Note: DOT is here because <TODO: REASON>
[".", "DOT"],
["..", "DOTDOT"],
["...", "DOTDOTDOT"],
["..=", "DOTDOTEQ"],