rust/crates/parser/src/input.rs

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

89 lines
2.6 KiB
Rust
Raw Normal View History

2021-12-25 12:59:02 -06:00
//! See [`Input`].
2021-12-12 10:06:40 -06:00
2021-12-12 08:58:45 -06:00
use crate::SyntaxKind;
2021-11-14 07:47:13 -06:00
2021-11-14 13:13:44 -06:00
#[allow(non_camel_case_types)]
2021-11-14 07:47:13 -06:00
type bits = u64;
2021-12-25 12:59:02 -06:00
/// Input for the parser -- a sequence of tokens.
2021-11-14 07:47:13 -06:00
///
2021-12-25 12:59:02 -06:00
/// As of now, parser doesn't have access to the *text* of the tokens, and makes
/// decisions based solely on their classification. Unlike `LexerToken`, the
/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
///
/// Struct of arrays internally, but this shouldn't really matter.
2021-11-14 07:47:13 -06:00
#[derive(Default)]
2021-12-25 12:59:02 -06:00
pub struct Input {
2021-11-14 07:47:13 -06:00
kind: Vec<SyntaxKind>,
joint: Vec<bits>,
2021-12-12 10:32:04 -06:00
contextual_kind: Vec<SyntaxKind>,
2021-11-14 07:47:13 -06:00
}
2021-12-12 10:36:14 -06:00
/// `pub` impl used by callers to create `Tokens`.
2021-12-25 12:59:02 -06:00
impl Input {
2021-12-12 10:17:04 -06:00
#[inline]
2021-12-12 09:38:49 -06:00
pub fn push(&mut self, kind: SyntaxKind) {
self.push_impl(kind, SyntaxKind::EOF)
}
2021-12-12 10:36:14 -06:00
#[inline]
pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
self.push_impl(SyntaxKind::IDENT, contextual_kind)
}
2021-12-12 10:06:40 -06:00
/// Sets jointness for the last token we've pushed.
///
/// This is a separate API rather than an argument to the `push` to make it
/// convenient both for textual and mbe tokens. With text, you know whether
/// the *previous* token was joint, with mbe, you know whether the *current*
/// one is joint. This API allows for styles of usage:
///
/// ```
/// // In text:
/// tokens.was_joint(prev_joint);
/// tokens.push(curr);
///
/// // In MBE:
/// token.push(curr);
/// tokens.push(curr_joint)
/// ```
2021-12-12 10:17:04 -06:00
#[inline]
2021-12-12 10:22:37 -06:00
pub fn was_joint(&mut self) {
2021-12-12 10:36:14 -06:00
let n = self.len() - 1;
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] |= 1 << b_idx;
2021-11-14 07:47:13 -06:00
}
2021-12-12 10:17:04 -06:00
#[inline]
2021-12-12 10:32:04 -06:00
fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
2021-11-14 07:47:13 -06:00
let idx = self.len();
if idx % (bits::BITS as usize) == 0 {
self.joint.push(0);
}
self.kind.push(kind);
2021-12-12 10:32:04 -06:00
self.contextual_kind.push(contextual_kind);
2021-11-14 07:47:13 -06:00
}
2021-12-12 10:31:32 -06:00
}
2021-11-14 12:37:10 -06:00
2021-12-12 10:36:14 -06:00
/// pub(crate) impl used by the parser to consume `Tokens`.
2021-12-25 12:59:02 -06:00
impl Input {
2021-12-12 10:31:32 -06:00
pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
}
pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
2021-12-12 10:32:04 -06:00
self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
2021-12-12 10:31:32 -06:00
}
pub(crate) fn is_joint(&self, n: usize) -> bool {
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] & 1 << b_idx != 0
2021-11-14 07:47:13 -06:00
}
}
2021-12-12 10:36:14 -06:00
2021-12-25 12:59:02 -06:00
impl Input {
2021-12-12 10:36:14 -06:00
fn bit_index(&self, n: usize) -> (usize, usize) {
let idx = n / (bits::BITS as usize);
let b_idx = n % (bits::BITS as usize);
(idx, b_idx)
}
fn len(&self) -> usize {
self.kind.len()
}
}