2021-05-22 08:53:47 -05:00
|
|
|
//! Stateful iteration over token trees.
|
|
|
|
//!
|
|
|
|
//! We use this as the source of tokens for parser.
|
2021-01-04 10:22:42 -06:00
|
|
|
use crate::{Leaf, Subtree, TokenTree};
|
2019-05-21 23:30:36 -05:00
|
|
|
|
|
|
|
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
|
|
|
|
struct EntryId(usize);
|
|
|
|
|
|
|
|
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
|
|
|
|
struct EntryPtr(EntryId, usize);
|
|
|
|
|
|
|
|
/// Internal type which is used instead of `TokenTree` to represent a token tree
|
|
|
|
/// within a `TokenBuffer`.
|
|
|
|
#[derive(Debug)]
|
2019-06-02 11:54:33 -05:00
|
|
|
enum Entry<'t> {
|
2019-05-21 23:30:36 -05:00
|
|
|
// Mimicking types from proc-macro.
|
2021-01-04 10:22:42 -06:00
|
|
|
Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId),
|
2019-06-02 11:54:33 -05:00
|
|
|
Leaf(&'t TokenTree),
|
2019-05-21 23:30:36 -05:00
|
|
|
// End entries contain a pointer to the entry from the containing
|
|
|
|
// token tree, or None if this is the outermost level.
|
|
|
|
End(Option<EntryPtr>),
|
|
|
|
}
|
|
|
|
|
|
|
|
/// A token tree buffer
|
|
|
|
/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
|
|
|
|
#[derive(Debug)]
|
2019-06-02 11:54:33 -05:00
|
|
|
pub struct TokenBuffer<'t> {
|
|
|
|
buffers: Vec<Box<[Entry<'t>]>>,
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|
|
|
|
|
2021-01-04 10:22:42 -06:00
|
|
|
trait TokenList<'a> {
|
|
|
|
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>);
|
|
|
|
}
|
2019-05-21 23:30:36 -05:00
|
|
|
|
2021-01-04 10:22:42 -06:00
|
|
|
impl<'a> TokenList<'a> for &'a [TokenTree] {
|
|
|
|
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
|
2020-05-16 11:24:17 -05:00
|
|
|
// Must contain everything in tokens and then the Entry::End
|
2021-01-04 10:22:42 -06:00
|
|
|
let start_capacity = self.len() + 1;
|
2020-05-16 11:24:17 -05:00
|
|
|
let mut entries = Vec::with_capacity(start_capacity);
|
2019-05-21 23:30:36 -05:00
|
|
|
let mut children = vec![];
|
2021-01-04 10:22:42 -06:00
|
|
|
for (idx, tt) in self.iter().enumerate() {
|
2019-05-21 23:30:36 -05:00
|
|
|
match tt {
|
2019-06-02 11:54:33 -05:00
|
|
|
TokenTree::Leaf(_) => {
|
|
|
|
entries.push(Entry::Leaf(tt));
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|
|
|
|
TokenTree::Subtree(subtree) => {
|
|
|
|
entries.push(Entry::End(None));
|
2021-01-04 10:22:42 -06:00
|
|
|
children.push((idx, (subtree, Some(tt))));
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-01-04 10:22:42 -06:00
|
|
|
(children, entries)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> TokenList<'a> for &'a Subtree {
|
|
|
|
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
|
|
|
|
// Must contain everything in tokens and then the Entry::End
|
|
|
|
let mut entries = vec![];
|
|
|
|
let mut children = vec![];
|
|
|
|
entries.push(Entry::End(None));
|
|
|
|
children.push((0usize, (*self, None)));
|
|
|
|
(children, entries)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'t> TokenBuffer<'t> {
|
|
|
|
pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
|
|
|
|
Self::new(tokens)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> {
|
|
|
|
Self::new(subtree)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> {
|
|
|
|
let mut buffers = vec![];
|
|
|
|
let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
|
|
|
|
assert_eq!(idx, 0);
|
|
|
|
TokenBuffer { buffers }
|
|
|
|
}
|
|
|
|
|
|
|
|
fn new_inner<T: TokenList<'t>>(
|
|
|
|
tokens: T,
|
|
|
|
buffers: &mut Vec<Box<[Entry<'t>]>>,
|
|
|
|
next: Option<EntryPtr>,
|
|
|
|
) -> usize {
|
|
|
|
let (children, mut entries) = tokens.entries();
|
2019-05-21 23:30:36 -05:00
|
|
|
|
|
|
|
entries.push(Entry::End(next));
|
|
|
|
let res = buffers.len();
|
|
|
|
buffers.push(entries.into_boxed_slice());
|
|
|
|
|
2019-06-02 11:54:33 -05:00
|
|
|
for (child_idx, (subtree, tt)) in children {
|
2019-05-21 23:30:36 -05:00
|
|
|
let idx = TokenBuffer::new_inner(
|
2021-01-04 10:22:42 -06:00
|
|
|
subtree.token_trees.as_slice(),
|
2019-05-21 23:30:36 -05:00
|
|
|
buffers,
|
|
|
|
Some(EntryPtr(EntryId(res), child_idx + 1)),
|
|
|
|
);
|
2021-01-04 10:22:42 -06:00
|
|
|
buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, subtree, EntryId(idx));
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
res
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Creates a cursor referencing the first token in the buffer and able to
|
|
|
|
/// traverse until the end of the buffer.
|
2021-05-22 08:53:47 -05:00
|
|
|
pub fn begin(&self) -> Cursor<'_> {
|
2019-05-21 23:30:36 -05:00
|
|
|
Cursor::create(self, EntryPtr(EntryId(0), 0))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry> {
|
|
|
|
let id = ptr.0;
|
|
|
|
self.buffers[id.0].get(ptr.1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-04 10:22:42 -06:00
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum TokenTreeRef<'a> {
|
|
|
|
Subtree(&'a Subtree, Option<&'a TokenTree>),
|
|
|
|
Leaf(&'a Leaf, &'a TokenTree),
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> TokenTreeRef<'a> {
|
|
|
|
pub fn cloned(&self) -> TokenTree {
|
|
|
|
match &self {
|
|
|
|
TokenTreeRef::Subtree(subtree, tt) => match tt {
|
|
|
|
Some(it) => (*it).clone(),
|
|
|
|
None => (*subtree).clone().into(),
|
|
|
|
},
|
|
|
|
TokenTreeRef::Leaf(_, tt) => (*tt).clone(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-13 23:57:10 -05:00
|
|
|
/// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125>
|
2019-05-21 23:30:36 -05:00
|
|
|
#[derive(Copy, Clone, Debug)]
|
|
|
|
pub struct Cursor<'a> {
|
2019-06-02 11:54:33 -05:00
|
|
|
buffer: &'a TokenBuffer<'a>,
|
2019-05-21 23:30:36 -05:00
|
|
|
ptr: EntryPtr,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> PartialEq for Cursor<'a> {
|
|
|
|
fn eq(&self, other: &Cursor) -> bool {
|
|
|
|
self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> Eq for Cursor<'a> {}
|
|
|
|
|
|
|
|
impl<'a> Cursor<'a> {
|
|
|
|
/// Check whether it is eof
|
|
|
|
pub fn eof(self) -> bool {
|
2020-06-27 20:02:03 -05:00
|
|
|
matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None)))
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
/// If the cursor is pointing at the end of a subtree, returns
|
|
|
|
/// the parent subtree
|
2019-11-02 17:19:08 -05:00
|
|
|
pub fn end(self) -> Option<&'a Subtree> {
|
2019-05-21 23:30:36 -05:00
|
|
|
match self.entry() {
|
|
|
|
Some(Entry::End(Some(ptr))) => {
|
|
|
|
let idx = ptr.1;
|
2021-01-04 10:22:42 -06:00
|
|
|
if let Some(Entry::Subtree(_, subtree, _)) =
|
2019-05-21 23:30:36 -05:00
|
|
|
self.buffer.entry(&EntryPtr(ptr.0, idx - 1))
|
|
|
|
{
|
|
|
|
return Some(subtree);
|
|
|
|
}
|
|
|
|
None
|
|
|
|
}
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-02 17:19:08 -05:00
|
|
|
fn entry(self) -> Option<&'a Entry<'a>> {
|
2019-05-21 23:30:36 -05:00
|
|
|
self.buffer.entry(&self.ptr)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// If the cursor is pointing at a `Subtree`, returns
|
|
|
|
/// a cursor into that subtree
|
|
|
|
pub fn subtree(self) -> Option<Cursor<'a>> {
|
|
|
|
match self.entry() {
|
2021-01-04 10:22:42 -06:00
|
|
|
Some(Entry::Subtree(_, _, entry_id)) => {
|
2019-05-21 23:30:36 -05:00
|
|
|
Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
|
|
|
|
}
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// If the cursor is pointing at a `TokenTree`, returns it
|
2021-01-04 10:22:42 -06:00
|
|
|
pub fn token_tree(self) -> Option<TokenTreeRef<'a>> {
|
2019-05-21 23:30:36 -05:00
|
|
|
match self.entry() {
|
2021-01-04 10:22:42 -06:00
|
|
|
Some(Entry::Leaf(tt)) => match tt {
|
|
|
|
TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, *tt)),
|
|
|
|
TokenTree::Subtree(subtree) => Some(TokenTreeRef::Subtree(subtree, Some(tt))),
|
|
|
|
},
|
|
|
|
Some(Entry::Subtree(tt, subtree, _)) => Some(TokenTreeRef::Subtree(subtree, *tt)),
|
2019-05-21 23:30:36 -05:00
|
|
|
Some(Entry::End(_)) => None,
|
|
|
|
None => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn create(buffer: &'a TokenBuffer, ptr: EntryPtr) -> Cursor<'a> {
|
|
|
|
Cursor { buffer, ptr }
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Bump the cursor
|
|
|
|
pub fn bump(self) -> Cursor<'a> {
|
|
|
|
if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) {
|
|
|
|
if let Some(exit) = exit {
|
|
|
|
Cursor::create(self.buffer, *exit)
|
|
|
|
} else {
|
|
|
|
self
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
Cursor::create(self.buffer, EntryPtr(self.ptr.0, self.ptr.1 + 1))
|
|
|
|
}
|
|
|
|
}
|
2019-05-27 10:33:23 -05:00
|
|
|
|
|
|
|
/// Bump the cursor, if it is a subtree, returns
|
|
|
|
/// a cursor into that subtree
|
|
|
|
pub fn bump_subtree(self) -> Cursor<'a> {
|
|
|
|
match self.entry() {
|
2021-01-04 10:22:42 -06:00
|
|
|
Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
|
2019-05-27 10:33:23 -05:00
|
|
|
_ => self.bump(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Check whether it is a top level
|
|
|
|
pub fn is_root(&self) -> bool {
|
|
|
|
let entry_id = self.ptr.0;
|
2019-06-03 09:01:27 -05:00
|
|
|
entry_id.0 == 0
|
2019-05-27 10:33:23 -05:00
|
|
|
}
|
2019-05-21 23:30:36 -05:00
|
|
|
}
|