Clarify and correct comment about multi_byte_tokens
This commit is contained in:
parent
3d3026dc60
commit
89cf7d8743
@ -25,9 +25,9 @@ Grammar(
|
||||
["^", "CARET"],
|
||||
["%", "PERCENT"],
|
||||
],
|
||||
// TODO: Confirm surmision: the tokens which cannot be recorded in a single UTF-8 byte
|
||||
// Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT)
|
||||
multi_byte_tokens: [
|
||||
[".", "DOT"], // Note: DOT is here because <TODO: REASON>
|
||||
[".", "DOT"],
|
||||
["..", "DOTDOT"],
|
||||
["...", "DOTDOTDOT"],
|
||||
["..=", "DOTDOTEQ"],
|
||||
|
Loading…
x
Reference in New Issue
Block a user