mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Infect mbe crate with generic span type parameter

Browse Source
This commit is contained in:
Lukas Wirth 2023-06-29 11:12:48 +02:00
parent 2ee17bc5f2
commit 83f91f61b1
10 changed files with 362 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
crates/mbe/src/benchmark.rs
View File

@ -6,10 +6,11 @@
AstNode, SmolStr,
};
use test_utils::{bench, bench_fixture, skip_slow_tests};
use tt::{Span, TokenId};
use crate::{
parser::{MetaVarKind, Op, RepeatKind, Separator},
syntax_node_to_token_tree, tt, DeclarativeMacro,
syntax_node_to_token_tree, DeclarativeMacro,
};
#[test]
@ -54,7 +55,7 @@ fn macro_rules_fixtures() -> FxHashMap<String, DeclarativeMacro> {
.collect()
}
fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree<TokenId>> {
let fixture = bench_fixture::numerous_macro_rules();
let source_file = ast::SourceFile::parse(&fixture).ok().unwrap();
@ -71,7 +72,9 @@ fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
}
/// Generate random invocation fixtures from rules
fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(String, tt::Subtree)> {
fn invocation_fixtures(
rules: &FxHashMap<String, DeclarativeMacro>,
) -> Vec<(String, tt::Subtree<TokenId>)> {
let mut seed = 123456789;
let mut res = Vec::new();
@ -93,8 +96,8 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
loop {
let mut subtree = tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
open: tt::TokenId::DUMMY,
close: tt::TokenId::DUMMY,
kind: tt::DelimiterKind::Invisible,
},
token_trees: vec![],
@ -116,7 +119,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
}
return res;
fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) {
fn collect_from_op(op: &Op<TokenId>, parent: &mut tt::Subtree<TokenId>, seed: &mut usize) {
return match op {
Op::Var { kind, .. } => match kind.as_ref() {
Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")),
@ -202,36 +205,30 @@ fn rand(seed: &mut usize) -> usize {
*seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c);
*seed
}
fn make_ident(ident: &str) -> tt::TokenTree {
tt::Leaf::Ident(tt::Ident {
span: tt::TokenId::unspecified(),
text: SmolStr::new(ident),
})
.into()
fn make_ident(ident: &str) -> tt::TokenTree<TokenId> {
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::DUMMY, text: SmolStr::new(ident) })
.into()
}
fn make_punct(char: char) -> tt::TokenTree {
fn make_punct(char: char) -> tt::TokenTree<TokenId> {
tt::Leaf::Punct(tt::Punct {
span: tt::TokenId::unspecified(),
span: tt::TokenId::DUMMY,
char,
spacing: tt::Spacing::Alone,
})
.into()
}
fn make_literal(lit: &str) -> tt::TokenTree {
tt::Leaf::Literal(tt::Literal {
span: tt::TokenId::unspecified(),
text: SmolStr::new(lit),
})
.into()
fn make_literal(lit: &str) -> tt::TokenTree<TokenId> {
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::DUMMY, text: SmolStr::new(lit) })
.into()
}
fn make_subtree(
kind: tt::DelimiterKind,
token_trees: Option<Vec<tt::TokenTree>>,
) -> tt::TokenTree {
token_trees: Option<Vec<tt::TokenTree<TokenId>>>,
) -> tt::TokenTree<TokenId> {
tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
open: tt::TokenId::DUMMY,
close: tt::TokenId::DUMMY,
kind,
},
token_trees: token_trees.unwrap_or_default(),

41
crates/mbe/src/expander.rs
View File

@ -7,15 +7,16 @@
use rustc_hash::FxHashMap;
use syntax::SmolStr;
use tt::Span;
use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult};
use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
pub(crate) fn expand_rules(
rules: &[crate::Rule],
input: &tt::Subtree,
pub(crate) fn expand_rules<S: Span>(
rules: &[crate::Rule<S>],
input: &tt::Subtree<S>,
is_2021: bool,
) -> ExpandResult<tt::Subtree> {
let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
) -> ExpandResult<tt::Subtree<S>> {
let mut match_: Option<(matcher::Match<S>, &crate::Rule<S>)> = None;
for rule in rules {
let new_match = matcher::match_(&rule.lhs, input, is_2021);
@ -47,7 +48,7 @@ pub(crate) fn expand_rules(
ExpandResult { value, err: match_.err.or(transcribe_err) }
} else {
ExpandResult::new(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] },
ExpandError::NoMatchingRule,
)
}
@ -98,23 +99,29 @@ pub(crate) fn expand_rules(
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but a `&[usize]`.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct Bindings {
inner: FxHashMap<SmolStr, Binding>,
#[derive(Debug, Clone, PartialEq, Eq)]
struct Bindings<S> {
inner: FxHashMap<SmolStr, Binding<S>>,
}
impl<S> Default for Bindings<S> {
fn default() -> Self {
Self { inner: Default::default() }
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Binding {
Fragment(Fragment),
Nested(Vec<Binding>),
enum Binding<S> {
Fragment(Fragment<S>),
Nested(Vec<Binding<S>>),
Empty,
Missing(MetaVarKind),
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Fragment {
enum Fragment<S> {
/// token fragments are just copy-pasted into the output
Tokens(tt::TokenTree),
Tokens(tt::TokenTree<S>),
/// Expr ast fragments are surrounded with `()` on insertion to preserve
/// precedence. Note that this impl is different from the one currently in
/// `rustc` -- `rustc` doesn't translate fragments into token trees at all.
@ -122,7 +129,7 @@ enum Fragment {
/// At one point in time, we tried to use "fake" delimiters here à la
/// proc-macro delimiter=none. As we later discovered, "none" delimiters are
/// tricky to handle in the parser, and rustc doesn't handle those either.
Expr(tt::TokenTree),
Expr(tt::TokenTree<S>),
/// There are roughly two types of paths: paths in expression context, where a
/// separator `::` between an identifier and its following generic argument list
/// is mandatory, and paths in type context, where `::` can be omitted.
@ -132,5 +139,5 @@ enum Fragment {
/// and is trasncribed as an expression-context path, verbatim transcription
/// would cause a syntax error. We need to fix it up just before transcribing;
/// see `transcriber::fix_up_and_push_path_tt()`.
Path(tt::TokenTree),
Path(tt::TokenTree<S>),
}

150
crates/mbe/src/expander/matcher.rs
View File

@ -63,21 +63,20 @@
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use tt::Span;
use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt,
tt_iter::TtIter,
ExpandError, MetaTemplate, ValueResult,
};
impl Bindings {
impl<S: Span> Bindings<S> {
fn push_optional(&mut self, name: &SmolStr) {
// FIXME: Do we have a better way to represent an empty token ?
// Insert an empty subtree for empty token
let tt =
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into();
let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }.into();
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
}
@ -85,14 +84,14 @@ fn push_empty(&mut self, name: &SmolStr) {
self.inner.insert(name.clone(), Binding::Empty);
}
fn bindings(&self) -> impl Iterator<Item = &Binding> {
fn bindings(&self) -> impl Iterator<Item = &Binding<S>> {
self.inner.values()
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(super) struct Match {
pub(super) bindings: Bindings,
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct Match<S> {
pub(super) bindings: Bindings<S>,
/// We currently just keep the first error and count the rest to compare matches.
pub(super) err: Option<ExpandError>,
pub(super) err_count: usize,
@ -102,7 +101,19 @@ pub(super) struct Match {
pub(super) bound_count: usize,
}
impl Match {
impl<S> Default for Match<S> {
fn default() -> Self {
Self {
bindings: Default::default(),
err: Default::default(),
err_count: Default::default(),
unmatched_tts: Default::default(),
bound_count: Default::default(),
}
}
}
impl<S> Match<S> {
fn add_err(&mut self, err: ExpandError) {
let prev_err = self.err.take();
self.err = prev_err.or(Some(err));
@ -111,12 +122,16 @@ fn add_err(&mut self, err: ExpandError) {
}
/// Matching errors are added to the `Match`.
pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree, is_2021: bool) -> Match {
pub(super) fn match_<S: Span>(
pattern: &MetaTemplate<S>,
input: &tt::Subtree<S>,
is_2021: bool,
) -> Match<S> {
let mut res = match_loop(pattern, input, is_2021);
res.bound_count = count(res.bindings.bindings());
return res;
fn count<'a>(bindings: impl Iterator<Item = &'a Binding>) -> usize {
fn count<'a, S: 'a>(bindings: impl Iterator<Item = &'a Binding<S>>) -> usize {
bindings
.map(|it| match it {
Binding::Fragment(_) => 1,
@ -129,10 +144,10 @@ fn count<'a>(bindings: impl Iterator<Item = &'a Binding>) -> usize {
}
#[derive(Debug, Clone)]
enum BindingKind {
enum BindingKind<S> {
Empty(SmolStr),
Optional(SmolStr),
Fragment(SmolStr, Fragment),
Fragment(SmolStr, Fragment<S>),
Missing(SmolStr, MetaVarKind),
Nested(usize, usize),
}
@ -146,13 +161,18 @@ enum LinkNode<T> {
Parent { idx: usize, len: usize },
}
#[derive(Default)]
struct BindingsBuilder {
nodes: Vec<Vec<LinkNode<Rc<BindingKind>>>>,
struct BindingsBuilder<S> {
nodes: Vec<Vec<LinkNode<Rc<BindingKind<S>>>>>,
nested: Vec<Vec<LinkNode<usize>>>,
}
impl BindingsBuilder {
impl<S> Default for BindingsBuilder<S> {
fn default() -> Self {
Self { nodes: Default::default(), nested: Default::default() }
}
}
impl<S: Span> BindingsBuilder<S> {
fn alloc(&mut self) -> BindingsIdx {
let idx = self.nodes.len();
self.nodes.push(Vec::new());
@ -189,7 +209,7 @@ fn push_optional(&mut self, idx: &mut BindingsIdx, var: &SmolStr) {
self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone()))));
}
fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) {
fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment<S>) {
self.nodes[idx.0]
.push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment))));
}
@ -210,11 +230,11 @@ fn push_default(&mut self, idx: &mut BindingsIdx) {
idx.0 = new_idx;
}
fn build(self, idx: &BindingsIdx) -> Bindings {
fn build(self, idx: &BindingsIdx) -> Bindings<S> {
self.build_inner(&self.nodes[idx.0])
}
fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind>>]) -> Bindings {
fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind<S>>>]) -> Bindings<S> {
let mut bindings = Bindings::default();
let mut nodes = Vec::new();
self.collect_nodes(link_nodes, &mut nodes);
@ -264,7 +284,7 @@ fn collect_nested_ref<'a>(
&'a self,
id: usize,
len: usize,
nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind>>]>,
nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind<S>>>]>,
) {
self.nested[id].iter().take(len).for_each(|it| match it {
LinkNode::Node(id) => nested_refs.push(&self.nodes[*id]),
@ -272,7 +292,7 @@ fn collect_nested_ref<'a>(
});
}
fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings>) {
fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings<S>>) {
let last = &self.nodes[idx];
let mut nested_refs: Vec<&[_]> = Vec::new();
self.nested[nested_idx].iter().for_each(|it| match *it {
@ -283,7 +303,7 @@ fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Binding
nested.extend(nested_refs.into_iter().map(|iter| self.build_inner(iter)));
}
fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind>) {
fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind<S>>) {
self.nodes[id].iter().take(len).for_each(|it| match it {
LinkNode::Node(it) => nodes.push(it),
LinkNode::Parent { idx, len } => self.collect_nodes_ref(*idx, *len, nodes),
@ -292,8 +312,8 @@ fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a Bi
fn collect_nodes<'a>(
&'a self,
link_nodes: &'a [LinkNode<Rc<BindingKind>>],
nodes: &mut Vec<&'a BindingKind>,
link_nodes: &'a [LinkNode<Rc<BindingKind<S>>>],
nodes: &mut Vec<&'a BindingKind<S>>,
) {
link_nodes.iter().for_each(|it| match it {
LinkNode::Node(it) => nodes.push(it),
@ -303,22 +323,22 @@ fn collect_nodes<'a>(
}
#[derive(Debug, Clone)]
struct MatchState<'t> {
struct MatchState<'t, S> {
/// The position of the "dot" in this matcher
dot: OpDelimitedIter<'t>,
dot: OpDelimitedIter<'t, S>,
/// Token subtree stack
/// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
/// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
/// that where the bottom of the stack is the outermost matcher.
stack: SmallVec<[OpDelimitedIter<'t>; 4]>,
stack: SmallVec<[OpDelimitedIter<'t, S>; 4]>,
/// The "parent" matcher position if we are in a repetition. That is, the matcher position just
/// before we enter the repetition.
up: Option<Box<MatchState<'t>>>,
up: Option<Box<MatchState<'t, S>>>,
/// The separator if we are in a repetition.
sep: Option<Separator>,
sep: Option<Separator<S>>,
/// The KleeneOp of this sequence if we are in a repetition.
sep_kind: Option<RepeatKind>,
@ -330,7 +350,7 @@ struct MatchState<'t> {
bindings: BindingsIdx,
/// Cached result of meta variable parsing
meta_result: Option<(TtIter<'t>, ExpandResult<Option<Fragment>>)>,
meta_result: Option<(TtIter<'t, S>, ExpandResult<Option<Fragment<S>>>)>,
/// Is error occurred in this state, will `poised` to "parent"
is_error: bool,
@ -355,16 +375,16 @@ struct MatchState<'t> {
/// - `bb_items`: the set of items that are waiting for the black-box parser.
/// - `error_items`: the set of items in errors, used for error-resilient parsing
#[inline]
fn match_loop_inner<'t>(
src: TtIter<'t>,
stack: &[TtIter<'t>],
res: &mut Match,
bindings_builder: &mut BindingsBuilder,
cur_items: &mut SmallVec<[MatchState<'t>; 1]>,
bb_items: &mut SmallVec<[MatchState<'t>; 1]>,
next_items: &mut Vec<MatchState<'t>>,
eof_items: &mut SmallVec<[MatchState<'t>; 1]>,
error_items: &mut SmallVec<[MatchState<'t>; 1]>,
fn match_loop_inner<'t, S: Span>(
src: TtIter<'t, S>,
stack: &[TtIter<'t, S>],
res: &mut Match<S>,
bindings_builder: &mut BindingsBuilder<S>,
cur_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
bb_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
next_items: &mut Vec<MatchState<'t, S>>,
eof_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
error_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
is_2021: bool,
) {
macro_rules! try_push {
@ -468,7 +488,7 @@ macro_rules! try_push {
if let Ok(subtree) = src.clone().expect_subtree() {
if subtree.delimiter.kind == delimiter.kind {
item.stack.push(item.dot);
item.dot = tokens.iter_delimited(Some(delimiter));
item.dot = tokens.iter_delimited(Some(*delimiter));
cur_items.push(item);
}
}
@ -587,9 +607,9 @@ macro_rules! try_push {
}
}
fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match {
fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021: bool) -> Match<S> {
let mut src = TtIter::new(src);
let mut stack: SmallVec<[TtIter<'_>; 1]> = SmallVec::new();
let mut stack: SmallVec<[TtIter<'_, S>; 1]> = SmallVec::new();
let mut res = Match::default();
let mut error_recover_item = None;
@ -736,11 +756,11 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match
}
}
fn match_meta_var(
fn match_meta_var<S: Span>(
kind: MetaVarKind,
input: &mut TtIter<'_>,
input: &mut TtIter<'_, S>,
is_2021: bool,
) -> ExpandResult<Option<Fragment>> {
) -> ExpandResult<Option<Fragment<S>>> {
let fragment = match kind {
MetaVarKind::Path => {
return input
@ -811,7 +831,7 @@ fn match_meta_var(
input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens))
}
fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) {
fn collect_vars<S: Span>(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate<S>) {
for op in pattern.iter() {
match op {
Op::Var { name, .. } => collector_fun(name.clone()),
@ -824,38 +844,38 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
}
}
}
impl MetaTemplate {
fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> {
impl<S: Span> MetaTemplate<S> {
fn iter_delimited(&self, delimited: Option<tt::Delimiter<S>>) -> OpDelimitedIter<'_, S> {
OpDelimitedIter {
inner: &self.0,
idx: 0,
delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED),
delimited: delimited.unwrap_or(tt::Delimiter::UNSPECIFIED),
}
}
}
#[derive(Debug, Clone, Copy)]
enum OpDelimited<'a> {
Op(&'a Op),
enum OpDelimited<'a, S> {
Op(&'a Op<S>),
Open,
Close,
}
#[derive(Debug, Clone, Copy)]
struct OpDelimitedIter<'a> {
inner: &'a [Op],
delimited: &'a tt::Delimiter,
struct OpDelimitedIter<'a, S> {
inner: &'a [Op<S>],
delimited: tt::Delimiter<S>,
idx: usize,
}
impl<'a> OpDelimitedIter<'a> {
impl<'a, S: Span> OpDelimitedIter<'a, S> {
fn is_eof(&self) -> bool {
let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
self.idx >= len
}
fn peek(&self) -> Option<OpDelimited<'a>> {
fn peek(&self) -> Option<OpDelimited<'a, S>> {
match self.delimited.kind {
tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
_ => match self.idx {
@ -871,8 +891,8 @@ fn reset(&self) -> Self {
}
}
impl<'a> Iterator for OpDelimitedIter<'a> {
type Item = OpDelimited<'a>;
impl<'a, S: Span> Iterator for OpDelimitedIter<'a, S> {
type Item = OpDelimited<'a, S>;
fn next(&mut self) -> Option<Self::Item> {
let res = self.peek();
@ -888,8 +908,8 @@ fn size_hint(&self) -> (usize, Option<usize>) {
}
}
impl TtIter<'_> {
fn expect_separator(&mut self, separator: &Separator) -> bool {
impl<S: Span> TtIter<'_, S> {
fn expect_separator(&mut self, separator: &Separator<S>) -> bool {
let mut fork = self.clone();
let ok = match separator {
Separator::Ident(lhs) => match fork.expect_ident_or_underscore() {
@ -919,7 +939,7 @@ fn expect_separator(&mut self, separator: &Separator) -> bool {
ok
}
fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
fn expect_tt(&mut self) -> Result<tt::TokenTree<S>, ()> {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) {
if punct.char == '\'' {
self.expect_lifetime()
@ -936,7 +956,7 @@ fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
}
}
fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
fn expect_lifetime(&mut self) -> Result<tt::TokenTree<S>, ()> {
let punct = self.expect_single_punct()?;
if punct.char != '\'' {
return Err(());
@ -953,7 +973,7 @@ fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
.into())
}
fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
fn eat_char(&mut self, c: char) -> Option<tt::TokenTree<S>> {
let mut fork = self.clone();
match fork.expect_char(c) {
Ok(_) => {

99
crates/mbe/src/expander/transcriber.rs
View File

@ -2,20 +2,20 @@
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use syntax::SmolStr;
use tt::{Delimiter, Span};
use crate::{
expander::{Binding, Bindings, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt::{self, Delimiter},
CountError, ExpandError, ExpandResult, MetaTemplate,
};
impl Bindings {
impl<S: Span> Bindings<S> {
fn contains(&self, name: &str) -> bool {
self.inner.contains_key(name)
}
fn get(&self, name: &str) -> Result<&Binding, ExpandError> {
fn get(&self, name: &str) -> Result<&Binding<S>, ExpandError> {
match self.inner.get(name) {
Some(binding) => Ok(binding),
None => Err(ExpandError::binding_error(format!("could not find binding `{name}`"))),
@ -26,7 +26,7 @@ fn get_fragment(
&self,
name: &str,
nesting: &mut [NestingState],
) -> Result<Fragment, ExpandError> {
) -> Result<Fragment<S>, ExpandError> {
macro_rules! binding_err {
($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) };
}
@ -54,15 +54,15 @@ macro_rules! binding_err {
Binding::Missing(it) => Ok(match it {
MetaVarKind::Stmt => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
span: tt::TokenId::unspecified(),
span: S::DUMMY,
char: ';',
spacing: tt::Spacing::Alone,
})))
}
MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
open: S::DUMMY,
close: S::DUMMY,
kind: tt::DelimiterKind::Brace,
},
token_trees: vec![],
@ -82,19 +82,19 @@ macro_rules! binding_err {
| MetaVarKind::Ident => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("missing"),
span: tt::TokenId::unspecified(),
span: S::DUMMY,
})))
}
MetaVarKind::Lifetime => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("'missing"),
span: tt::TokenId::unspecified(),
span: S::DUMMY,
})))
}
MetaVarKind::Literal => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("\"missing\""),
span: tt::TokenId::unspecified(),
span: S::DUMMY,
})))
}
}),
@ -108,12 +108,12 @@ macro_rules! binding_err {
}
}
pub(super) fn transcribe(
template: &MetaTemplate,
bindings: &Bindings,
) -> ExpandResult<tt::Subtree> {
pub(super) fn transcribe<S: Span>(
template: &MetaTemplate<S>,
bindings: &Bindings<S>,
) -> ExpandResult<tt::Subtree<S>> {
let mut ctx = ExpandCtx { bindings, nesting: Vec::new() };
let mut arena: Vec<tt::TokenTree> = Vec::new();
let mut arena: Vec<tt::TokenTree<S>> = Vec::new();
expand_subtree(&mut ctx, template, None, &mut arena)
}
@ -129,17 +129,17 @@ struct NestingState {
}
#[derive(Debug)]
struct ExpandCtx<'a> {
bindings: &'a Bindings,
struct ExpandCtx<'a, S> {
bindings: &'a Bindings<S>,
nesting: Vec<NestingState>,
}
fn expand_subtree(
ctx: &mut ExpandCtx<'_>,
template: &MetaTemplate,
delimiter: Option<Delimiter>,
arena: &mut Vec<tt::TokenTree>,
) -> ExpandResult<tt::Subtree> {
fn expand_subtree<S: Span>(
ctx: &mut ExpandCtx<'_, S>,
template: &MetaTemplate<S>,
delimiter: Option<Delimiter<S>>,
arena: &mut Vec<tt::TokenTree<S>>,
) -> ExpandResult<tt::Subtree<S>> {
// remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
let start_elements = arena.len();
let mut err = None;
@ -180,7 +180,7 @@ fn expand_subtree(
arena.push(
tt::Leaf::Literal(tt::Literal {
text: index.to_string().into(),
span: tt::TokenId::unspecified(),
span: S::DUMMY,
})
.into(),
);
@ -237,11 +237,8 @@ fn expand_subtree(
}
};
arena.push(
tt::Leaf::Literal(tt::Literal {
text: c.to_string().into(),
span: tt::TokenId::unspecified(),
})
.into(),
tt::Leaf::Literal(tt::Literal { text: c.to_string().into(), span: S::DUMMY })
.into(),
);
}
}
@ -257,7 +254,11 @@ fn expand_subtree(
}
}
fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
fn expand_var<S: Span>(
ctx: &mut ExpandCtx<'_, S>,
v: &SmolStr,
id: S,
) -> ExpandResult<Fragment<S>> {
// We already handle $crate case in mbe parser
debug_assert!(v != "crate");
@ -296,14 +297,14 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
}
}
fn expand_repeat(
ctx: &mut ExpandCtx<'_>,
template: &MetaTemplate,
fn expand_repeat<S: Span>(
ctx: &mut ExpandCtx<'_, S>,
template: &MetaTemplate<S>,
kind: RepeatKind,
separator: &Option<Separator>,
arena: &mut Vec<tt::TokenTree>,
) -> ExpandResult<Fragment> {
let mut buf: Vec<tt::TokenTree> = Vec::new();
separator: &Option<Separator<S>>,
arena: &mut Vec<tt::TokenTree<S>>,
) -> ExpandResult<Fragment<S>> {
let mut buf: Vec<tt::TokenTree<S>> = Vec::new();
ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a proper macro-by-example implementation
@ -342,7 +343,7 @@ fn expand_repeat(
continue;
}
t.delimiter = tt::Delimiter::unspecified();
t.delimiter = tt::Delimiter::UNSPECIFIED;
push_subtree(&mut buf, t);
if let Some(sep) = separator {
@ -376,7 +377,7 @@ fn expand_repeat(
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into();
let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: buf }.into();
if RepeatKind::OneOrMore == kind && counter == 0 {
return ExpandResult {
@ -387,14 +388,14 @@ fn expand_repeat(
ExpandResult { value: Fragment::Tokens(tt), err }
}
fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
fn push_fragment<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, fragment: Fragment<S>) {
match fragment {
Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => {
if tt.delimiter.kind == tt::DelimiterKind::Invisible {
tt.delimiter = tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
open: S::DUMMY,
close: S::DUMMY,
kind: tt::DelimiterKind::Parenthesis,
};
}
@ -405,7 +406,7 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
}
}
fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) {
match tt.delimiter.kind {
tt::DelimiterKind::Invisible => buf.extend(tt.token_trees),
_ => buf.push(tt.into()),
@ -415,7 +416,7 @@ fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
/// Inserts the path separator `::` between an identifier and its following generic
/// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why
/// we need this fixup.
fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
fn fix_up_and_push_path_tt<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, subtree: tt::Subtree<S>) {
stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible));
let mut prev_was_ident = false;
// Note that we only need to fix up the top-level `TokenTree`s because the
@ -432,7 +433,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
tt::Leaf::Punct(tt::Punct {
char: ':',
spacing: tt::Spacing::Joint,
span: tt::Span::unspecified(),
span: S::DUMMY,
})
.into(),
);
@ -440,7 +441,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
tt::Leaf::Punct(tt::Punct {
char: ':',
spacing: tt::Spacing::Alone,
span: tt::Span::unspecified(),
span: S::DUMMY,
})
.into(),
);
@ -453,9 +454,9 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
/// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth
/// defined by the metavar expression.
fn count(
ctx: &ExpandCtx<'_>,
binding: &Binding,
fn count<S>(
ctx: &ExpandCtx<'_, S>,
binding: &Binding<S>,
our_depth: usize,
count_depth: Option<usize>,
) -> Result<usize, CountError> {

31
crates/mbe/src/lib.rs
View File

@ -18,8 +18,8 @@
mod benchmark;
mod token_map;
use ::tt::token_id as tt;
use stdx::impl_from;
use tt::{Span, TokenId};
use std::fmt;
@ -28,8 +28,9 @@
tt_iter::TtIter,
};
pub use self::tt::{Delimiter, DelimiterKind, Punct};
// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces
pub use ::parser::TopEntryPoint;
pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::{
syntax_bridge::{
@ -125,7 +126,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DeclarativeMacro {
rules: Box<[Rule]>,
rules: Box<[Rule<TokenId>]>,
/// Highest id of the token we have in TokenMap
shift: Shift,
// This is used for correctly determining the behavior of the pat fragment
@ -135,23 +136,23 @@ pub struct DeclarativeMacro {
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct Rule {
lhs: MetaTemplate,
rhs: MetaTemplate,
struct Rule<S> {
lhs: MetaTemplate<S>,
rhs: MetaTemplate<S>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Shift(u32);
impl Shift {
pub fn new(tt: &tt::Subtree) -> Shift {
pub fn new(tt: &tt::Subtree<TokenId>) -> Shift {
// Note that TokenId is started from zero,
// We have to add 1 to prevent duplication.
let value = max_id(tt).map_or(0, |it| it + 1);
return Shift(value);
// Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> {
fn max_id(subtree: &tt::Subtree<TokenId>) -> Option<u32> {
let filter =
|tt: &_| match tt {
tt::TokenTree::Subtree(subtree) => {
@ -177,7 +178,7 @@ fn max_id(subtree: &tt::Subtree) -> Option<u32> {
}
/// Shift given TokenTree token id
pub fn shift_all(self, tt: &mut tt::Subtree) {
pub fn shift_all(self, tt: &mut tt::Subtree<TokenId>) {
for t in &mut tt.token_trees {
match t {
tt::TokenTree::Leaf(
@ -224,7 +225,7 @@ pub fn from_err(err: ParseError, is_2021: bool) -> DeclarativeMacro {
}
/// The old, `macro_rules! m {}` flavor.
pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
pub fn parse_macro_rules(tt: &tt::Subtree<TokenId>, is_2021: bool) -> DeclarativeMacro {
// Note: this parsing can be implemented using mbe machinery itself, by
// matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
// manually seems easier.
@ -260,7 +261,7 @@ pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
}
/// The new, unstable `macro m {}` flavor.
pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
pub fn parse_macro2(tt: &tt::Subtree<TokenId>, is_2021: bool) -> DeclarativeMacro {
let mut src = TtIter::new(tt);
let mut rules = Vec::new();
let mut err = None;
@ -310,7 +311,7 @@ pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
DeclarativeMacro { rules: rules.into_boxed_slice(), shift: Shift::new(tt), is_2021, err }
}
pub fn expand(&self, mut tt: tt::Subtree) -> ExpandResult<tt::Subtree> {
pub fn expand(&self, mut tt: tt::Subtree<TokenId>) -> ExpandResult<tt::Subtree<TokenId>> {
self.shift.shift_all(&mut tt);
expander::expand_rules(&self.rules, &tt, self.is_2021)
}
@ -335,8 +336,8 @@ pub fn shift(&self) -> Shift {
}
}
impl Rule {
fn parse(src: &mut TtIter<'_>, expect_arrow: bool) -> Result<Self, ParseError> {
impl<S: Span> Rule<S> {
fn parse(src: &mut TtIter<'_, S>, expect_arrow: bool) -> Result<Self, ParseError> {
let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
if expect_arrow {
src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
@ -351,7 +352,7 @@ fn parse(src: &mut TtIter<'_>, expect_arrow: bool) -> Result<Self, ParseError> {
}
}
fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
fn validate<S: Span>(pattern: &MetaTemplate<S>) -> Result<(), ParseError> {
for op in pattern.iter() {
match op {
Op::Subtree { tokens, .. } => validate(tokens)?,

68
crates/mbe/src/parser.rs
View File

@ -3,8 +3,9 @@
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use tt::Span;
use crate::{tt, tt_iter::TtIter, ParseError};
use crate::{tt_iter::TtIter, ParseError};
/// Consider
///
@ -20,22 +21,22 @@
/// Stuff to the right is a [`MetaTemplate`] template which is used to produce
/// output.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>);
pub(crate) struct MetaTemplate<S>(pub(crate) Box<[Op<S>]>);
impl MetaTemplate {
pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result<MetaTemplate, ParseError> {
impl<S: Span> MetaTemplate<S> {
pub(crate) fn parse_pattern(pattern: &tt::Subtree<S>) -> Result<Self, ParseError> {
MetaTemplate::parse(pattern, Mode::Pattern)
}
pub(crate) fn parse_template(template: &tt::Subtree) -> Result<MetaTemplate, ParseError> {
pub(crate) fn parse_template(template: &tt::Subtree<S>) -> Result<Self, ParseError> {
MetaTemplate::parse(template, Mode::Template)
}
pub(crate) fn iter(&self) -> impl Iterator<Item = &Op> {
pub(crate) fn iter(&self) -> impl Iterator<Item = &Op<S>> {
self.0.iter()
}
fn parse(tt: &tt::Subtree, mode: Mode) -> Result<MetaTemplate, ParseError> {
fn parse(tt: &tt::Subtree<S>, mode: Mode) -> Result<Self, ParseError> {
let mut src = TtIter::new(tt);
let mut res = Vec::new();
@ -49,16 +50,16 @@ fn parse(tt: &tt::Subtree, mode: Mode) -> Result<MetaTemplate, ParseError> {
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Op {
Var { name: SmolStr, kind: Option<MetaVarKind>, id: tt::TokenId },
Ignore { name: SmolStr, id: tt::TokenId },
pub(crate) enum Op<S> {
Var { name: SmolStr, kind: Option<MetaVarKind>, id: S },
Ignore { name: SmolStr, id: S },
Index { depth: usize },
Count { name: SmolStr, depth: Option<usize> },
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter },
Literal(tt::Literal),
Punct(SmallVec<[tt::Punct; 3]>),
Ident(tt::Ident),
Repeat { tokens: MetaTemplate<S>, kind: RepeatKind, separator: Option<Separator<S>> },
Subtree { tokens: MetaTemplate<S>, delimiter: tt::Delimiter<S> },
Literal(tt::Literal<S>),
Punct(SmallVec<[tt::Punct<S>; 3]>),
Ident(tt::Ident<S>),
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
@ -87,15 +88,15 @@ pub(crate) enum MetaVarKind {
}
#[derive(Clone, Debug, Eq)]
pub(crate) enum Separator {
Literal(tt::Literal),
Ident(tt::Ident),
Puncts(SmallVec<[tt::Punct; 3]>),
pub(crate) enum Separator<S> {
Literal(tt::Literal<S>),
Ident(tt::Ident<S>),
Puncts(SmallVec<[tt::Punct<S>; 3]>),
}
// Note that when we compare a Separator, we just care about its textual value.
impl PartialEq for Separator {
fn eq(&self, other: &Separator) -> bool {
impl<S> PartialEq for Separator<S> {
fn eq(&self, other: &Separator<S>) -> bool {
use Separator::*;
match (self, other) {
@ -117,11 +118,11 @@ enum Mode {
Template,
}
fn next_op(
first_peeked: &tt::TokenTree,
src: &mut TtIter<'_>,
fn next_op<S: Span>(
first_peeked: &tt::TokenTree<S>,
src: &mut TtIter<'_, S>,
mode: Mode,
) -> Result<Op, ParseError> {
) -> Result<Op<S>, ParseError> {
let res = match first_peeked {
tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
src.next().expect("first token already peeked");
@ -212,7 +213,10 @@ fn next_op(
Ok(res)
}
fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<MetaVarKind>, ParseError> {
fn eat_fragment_kind<S: Span>(
src: &mut TtIter<'_, S>,
mode: Mode,
) -> Result<Option<MetaVarKind>, ParseError> {
if let Mode::Pattern = mode {
src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
let ident = src
@ -240,11 +244,13 @@ fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<MetaVarK
Ok(None)
}
fn is_boolean_literal(lit: &tt::Literal) -> bool {
fn is_boolean_literal<S>(lit: &tt::Literal<S>) -> bool {
matches!(lit.text.as_str(), "true" | "false")
}
fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind), ParseError> {
fn parse_repeat<S: Span>(
src: &mut TtIter<'_, S>,
) -> Result<(Option<Separator<S>>, RepeatKind), ParseError> {
let mut separator = Separator::Puncts(SmallVec::new());
for tt in src {
let tt = match tt {
@ -281,7 +287,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind),
Err(ParseError::InvalidRepeat)
}
fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
fn parse_metavar_expr<S: Span>(src: &mut TtIter<'_, S>) -> Result<Op<S>, ()> {
let func = src.expect_ident()?;
let args = src.expect_subtree()?;
@ -314,7 +320,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
Ok(op)
}
fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {
fn parse_depth<S: Span>(src: &mut TtIter<'_, S>) -> Result<usize, ()> {
if src.len() == 0 {
Ok(0)
} else if let tt::Leaf::Literal(lit) = src.expect_literal()? {
@ -325,7 +331,7 @@ fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {
}
}
fn try_eat_comma(src: &mut TtIter<'_>) -> bool {
fn try_eat_comma<S: Span>(src: &mut TtIter<'_, S>) -> bool {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek_n(0) {
let _ = src.next();
return true;

54
crates/mbe/src/syntax_bridge.rs
View File

@ -8,23 +8,19 @@
SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
use crate::{
to_parser_input::to_parser_input,
tt::{
self,
buffer::{Cursor, TokenBuffer},
},
tt_iter::TtIter,
TokenMap,
use tt::{
buffer::{Cursor, TokenBuffer},
TokenId,
};
use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
#[cfg(test)]
mod tests;
/// Convert the syntax node to a `TokenTree` (what macro
/// will consume).
pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) {
pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree<TokenId>, TokenMap) {
let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications(
node,
Default::default(),
@ -43,7 +39,7 @@ pub fn syntax_node_to_token_tree_with_modifications(
next_id: u32,
replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
) -> (tt::Subtree, TokenMap, u32) {
) -> (tt::Subtree<TokenId>, TokenMap, u32) {
let global_offset = node.text_range().start();
let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append);
let subtree = convert_tokens(&mut c);
@ -108,7 +104,7 @@ pub struct SyntheticToken {
// * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
pub fn token_tree_to_syntax_node(
tt: &tt::Subtree,
tt: &tt::Subtree<TokenId>,
entry_point: parser::TopEntryPoint,
) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt {
@ -138,7 +134,7 @@ pub fn token_tree_to_syntax_node(
}
/// Convert a string to a `TokenTree`
pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree<TokenId>, TokenMap)> {
let lexed = parser::LexedStr::new(text);
if lexed.errors().next().is_some() {
return None;
@ -159,7 +155,7 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
}
/// Split token tree with separate expr: $($e:expr)SEP*
pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
pub fn parse_exprs_with_sep(tt: &tt::Subtree<TokenId>, sep: char) -> Vec<tt::Subtree<TokenId>> {
if tt.token_trees.is_empty() {
return Vec::new();
}
@ -195,9 +191,9 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
res
}
fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree<TokenId> {
struct StackEntry {
subtree: tt::Subtree,
subtree: tt::Subtree<TokenId>,
idx: usize,
open_range: TextRange,
}
@ -296,7 +292,7 @@ macro_rules! make_leaf {
.into()
};
}
let leaf: tt::Leaf = match kind {
let leaf: tt::Leaf<TokenId> = match kind {
T![true] | T![false] => make_leaf!(Ident),
IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident),
@ -335,7 +331,7 @@ macro_rules! make_leaf {
let parent = stack.last_mut();
conv.id_alloc().close_delim(entry.idx, None);
let leaf: tt::Leaf = tt::Punct {
let leaf: tt::Leaf<TokenId> = tt::Punct {
span: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => '(',
@ -514,7 +510,7 @@ fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
fn convert_doc_comment(
token: &syntax::SyntaxToken,
span: tt::TokenId,
) -> Option<Vec<tt::TokenTree>> {
) -> Option<Vec<tt::TokenTree<TokenId>>> {
cov_mark::hit!(test_meta_doc_comments);
let comment = ast::Comment::cast(token.clone())?;
let doc = comment.kind().doc?;
@ -537,11 +533,11 @@ fn convert_doc_comment(
return Some(token_trees);
// Helper functions
fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree {
fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree<TokenId> {
tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), span }))
}
fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree {
fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree<TokenId> {
tt::TokenTree::from(tt::Leaf::from(tt::Punct {
char: c,
spacing: tt::Spacing::Alone,
@ -549,7 +545,7 @@ fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree {
}))
}
fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree {
fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree<TokenId> {
let lit = tt::Literal { text: doc_comment_text(comment), span };
tt::TokenTree::from(tt::Leaf::from(lit))
@ -636,7 +632,7 @@ fn convert_doc_comment(
&self,
token: &Self::Token,
span: tt::TokenId,
) -> Option<Vec<tt::TokenTree>>;
) -> Option<Vec<tt::TokenTree<TokenId>>>;
fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
@ -666,7 +662,11 @@ fn synthetic_id(&self, _ctx: &RawConverter<'_>) -> Option<SyntheticTokenId> {
impl TokenConverter for RawConverter<'_> {
type Token = usize;
fn convert_doc_comment(&self, &token: &usize, span: tt::TokenId) -> Option<Vec<tt::TokenTree>> {
fn convert_doc_comment(
&self,
&token: &usize,
span: tt::TokenId,
) -> Option<Vec<tt::TokenTree<TokenId>>> {
let text = self.lexed.text(token);
convert_doc_comment(&doc_comment(text), span)
}
@ -819,7 +819,7 @@ fn convert_doc_comment(
&self,
token: &Self::Token,
span: tt::TokenId,
) -> Option<Vec<tt::TokenTree>> {
) -> Option<Vec<tt::TokenTree<TokenId>>> {
convert_doc_comment(token.token()?, span)
}
@ -899,7 +899,7 @@ fn id_alloc(&mut self) -> &mut TokenIdAlloc {
struct TtTreeSink<'a> {
buf: String,
cursor: Cursor<'a>,
cursor: Cursor<'a, TokenId>,
open_delims: FxHashMap<tt::TokenId, TextSize>,
text_pos: TextSize,
inner: SyntaxTreeBuilder,
@ -907,7 +907,7 @@ struct TtTreeSink<'a> {
}
impl<'a> TtTreeSink<'a> {
fn new(cursor: Cursor<'a>) -> Self {
fn new(cursor: Cursor<'a, TokenId>) -> Self {
TtTreeSink {
buf: String::new(),
cursor,

4
crates/mbe/src/to_parser_input.rs
View File

@ -3,9 +3,9 @@
use syntax::{SyntaxKind, SyntaxKind::*, T};
use crate::tt::buffer::TokenBuffer;
use tt::{buffer::TokenBuffer, Span};
pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
pub(crate) fn to_parser_input<S: Span>(buffer: &TokenBuffer<'_, S>) -> parser::Input {
let mut res = parser::Input::default();
let mut current = buffer.begin();

35
crates/mbe/src/tt_iter.rs
View File

@ -3,16 +3,17 @@
use smallvec::{smallvec, SmallVec};
use syntax::SyntaxKind;
use tt::Span;
use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult};
use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult};
#[derive(Debug, Clone)]
pub(crate) struct TtIter<'a> {
pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
pub(crate) struct TtIter<'a, S> {
pub(crate) inner: std::slice::Iter<'a, tt::TokenTree<S>>,
}
impl<'a> TtIter<'a> {
pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
impl<'a, S: Span> TtIter<'a, S> {
pub(crate) fn new(subtree: &'a tt::Subtree<S>) -> TtIter<'a, S> {
TtIter { inner: subtree.token_trees.iter() }
}
@ -36,35 +37,35 @@ pub(crate) fn expect_any_char(&mut self, chars: &[char]) -> Result<(), ()> {
}
}
pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree<S>, ()> {
match self.next() {
Some(tt::TokenTree::Subtree(it)) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf<S>, ()> {
match self.next() {
Some(tt::TokenTree::Leaf(it)) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident<S>, ()> {
match self.expect_leaf()? {
tt::Leaf::Ident(it) if it.text != "_" => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> {
pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident<S>, ()> {
match self.expect_leaf()? {
tt::Leaf::Ident(it) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> {
pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf<S>, ()> {
let it = self.expect_leaf()?;
match it {
tt::Leaf::Literal(_) => Ok(it),
@ -73,7 +74,7 @@ pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> {
}
}
pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> {
pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct<S>, ()> {
match self.expect_leaf()? {
tt::Leaf::Punct(it) => Ok(it),
_ => Err(()),
@ -84,7 +85,7 @@ pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> {
///
/// This method currently may return a single quotation, which is part of lifetime ident and
/// conceptually not a punct in the context of mbe. Callers should handle this.
pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct; 3]>, ()> {
pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct<S>; 3]>, ()> {
let tt::TokenTree::Leaf(tt::Leaf::Punct(first)) = self.next().ok_or(())?.clone() else {
return Err(());
};
@ -126,7 +127,7 @@ pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> {
pub(crate) fn expect_fragment(
&mut self,
entry_point: parser::PrefixEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> {
) -> ExpandResult<Option<tt::TokenTree<S>>> {
let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
let parser_input = to_parser_input(&buffer);
let tree_traversal = entry_point.parse(&parser_input);
@ -181,13 +182,13 @@ pub(crate) fn expect_fragment(
ExpandResult { value: res, err }
}
pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> {
pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree<S>> {
self.inner.as_slice().get(n)
}
}
impl<'a> Iterator for TtIter<'a> {
type Item = &'a tt::TokenTree;
impl<'a, S> Iterator for TtIter<'a, S> {
type Item = &'a tt::TokenTree<S>;
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()
}
@ -197,4 +198,4 @@ fn size_hint(&self) -> (usize, Option<usize>) {
}
}
impl std::iter::ExactSizeIterator for TtIter<'_> {}
impl<S> std::iter::ExactSizeIterator for TtIter<'_, S> {}

162
crates/tt/src/lib.rs
View File

@ -47,70 +47,44 @@ pub mod buffer {
pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>;
pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>;
}
}
impl Delimiter {
pub const UNSPECIFIED: Self = Self {
open: TokenId::UNSPECIFIED,
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Invisible,
};
pub const fn unspecified() -> Self {
Self::UNSPECIFIED
}
}
impl Subtree {
pub const fn empty() -> Self {
Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
}
}
impl TokenTree {
pub const fn empty() -> Self {
Self::Subtree(Subtree::empty())
}
}
pub trait Span: std::fmt::Debug + Copy + Sized {
const DUMMY: Self;
fn is_dummy(&self) -> bool;
}
impl Span for TokenId {
const DUMMY: Self = TokenId(!0);
impl Subtree {
pub fn visit_ids(&mut self, f: &mut impl FnMut(TokenId) -> TokenId) {
self.delimiter.open = f(self.delimiter.open);
self.delimiter.close = f(self.delimiter.close);
self.token_trees.iter_mut().for_each(|tt| match tt {
crate::TokenTree::Leaf(leaf) => match leaf {
crate::Leaf::Literal(it) => it.span = f(it.span),
crate::Leaf::Punct(it) => it.span = f(it.span),
crate::Leaf::Ident(it) => it.span = f(it.span),
},
crate::TokenTree::Subtree(s) => s.visit_ids(f),
})
}
fn is_dummy(&self) -> bool {
*self == Self::DUMMY
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SyntaxContext(pub u32);
// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
// pub struct Span {
// pub id: TokenId,
// pub ctx: SyntaxContext,
// }
// pub type Span = (TokenId, SyntaxContext);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree<Span> {
Leaf(Leaf<Span>),
Subtree(Subtree<Span>),
pub enum TokenTree<S> {
Leaf(Leaf<S>),
Subtree(Subtree<S>),
}
impl_from!(Leaf<Span>, Subtree<Span> for TokenTree);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf<Span> {
Literal(Literal<Span>),
Punct(Punct<Span>),
Ident(Ident<Span>),
impl_from!(Leaf<S>, Subtree<S> for TokenTree);
impl<S: Span> TokenTree<S> {
pub const fn empty() -> Self {
Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] })
}
}
impl<Span> Leaf<Span> {
pub fn span(&self) -> &Span {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf<S> {
Literal(Literal<S>),
Punct(Punct<S>),
Ident(Ident<S>),
}
impl<S> Leaf<S> {
pub fn span(&self) -> &S {
match self {
Leaf::Literal(it) => &it.span,
Leaf::Punct(it) => &it.span,
@ -118,21 +92,49 @@ pub fn span(&self) -> &Span {
}
}
}
impl_from!(Literal<Span>, Punct<Span>, Ident<Span> for Leaf);
impl_from!(Literal<S>, Punct<S>, Ident<S> for Leaf);
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct Subtree<Span> {
pub delimiter: Delimiter<Span>,
pub token_trees: Vec<TokenTree<Span>>,
pub struct Subtree<S> {
pub delimiter: Delimiter<S>,
pub token_trees: Vec<TokenTree<S>>,
}
impl<S: Span> Subtree<S> {
pub const fn empty() -> Self {
Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
}
pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) {
self.delimiter.open = f(self.delimiter.open);
self.delimiter.close = f(self.delimiter.close);
self.token_trees.iter_mut().for_each(|tt| match tt {
crate::TokenTree::Leaf(leaf) => match leaf {
crate::Leaf::Literal(it) => it.span = f(it.span),
crate::Leaf::Punct(it) => it.span = f(it.span),
crate::Leaf::Ident(it) => it.span = f(it.span),
},
crate::TokenTree::Subtree(s) => s.visit_ids(f),
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Delimiter<Span> {
pub open: Span,
pub close: Span,
pub struct Delimiter<S> {
pub open: S,
pub close: S,
pub kind: DelimiterKind,
}
impl<S: Span> Delimiter<S> {
pub const UNSPECIFIED: Self =
Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible };
pub const fn unspecified() -> Self {
Self::UNSPECIFIED
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum DelimiterKind {
Parenthesis,
@ -142,16 +144,16 @@ pub enum DelimiterKind {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Literal<Span> {
pub struct Literal<S> {
pub text: SmolStr,
pub span: Span,
pub span: S,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Punct<Span> {
pub struct Punct<S> {
pub char: char,
pub spacing: Spacing,
pub span: Span,
pub span: S,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -162,9 +164,9 @@ pub enum Spacing {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
pub struct Ident<Span> {
pub struct Ident<S> {
pub text: SmolStr,
pub span: Span,
pub span: S,
}
impl<S> Ident<S> {
@ -173,9 +175,9 @@ pub fn new(text: impl Into<SmolStr>, span: S) -> Self {
}
}
fn print_debug_subtree<Span: fmt::Debug>(
fn print_debug_subtree<S: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
subtree: &Subtree<Span>,
subtree: &Subtree<S>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
@ -203,9 +205,9 @@ fn print_debug_subtree<Span: fmt::Debug>(
Ok(())
}
fn print_debug_token<Span: fmt::Debug>(
fn print_debug_token<S: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
tkn: &TokenTree<Span>,
tkn: &TokenTree<S>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
@ -231,13 +233,13 @@ fn print_debug_token<Span: fmt::Debug>(
Ok(())
}
impl<Span: fmt::Debug> fmt::Debug for Subtree<Span> {
impl<S: fmt::Debug> fmt::Debug for Subtree<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
print_debug_subtree(f, self, 0)
}
}
impl<Span> fmt::Display for TokenTree<Span> {
impl<S> fmt::Display for TokenTree<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
@ -246,7 +248,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
}
}
impl<Span> fmt::Display for Subtree<Span> {
impl<S> fmt::Display for Subtree<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (l, r) = match self.delimiter.kind {
DelimiterKind::Parenthesis => ("(", ")"),
@ -274,7 +276,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
}
}
impl<Span> fmt::Display for Leaf<Span> {
impl<S> fmt::Display for Leaf<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Leaf::Ident(it) => fmt::Display::fmt(it, f),
@ -284,25 +286,25 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
}
}
impl<Span> fmt::Display for Ident<Span> {
impl<S> fmt::Display for Ident<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl<Span> fmt::Display for Literal<Span> {
impl<S> fmt::Display for Literal<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl<Span> fmt::Display for Punct<Span> {
impl<S> fmt::Display for Punct<S> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.char, f)
}
}
impl<Span> Subtree<Span> {
impl<S> Subtree<S> {
/// Count the number of tokens recursively
pub fn count(&self) -> usize {
let children_count = self
@ -318,7 +320,7 @@ pub fn count(&self) -> usize {
}
}
impl<Span> Subtree<Span> {
impl<S> Subtree<S> {
/// A simple line string used for debugging
pub fn as_debug_string(&self) -> String {
let delim = match self.delimiter.kind {
@ -366,8 +368,8 @@ pub fn as_debug_string(&self) -> String {
pub mod buffer;
pub fn pretty<Span>(tkns: &[TokenTree<Span>]) -> String {
fn tokentree_to_text<Span>(tkn: &TokenTree<Span>) -> String {
pub fn pretty<S>(tkns: &[TokenTree<S>]) -> String {
fn tokentree_to_text<S>(tkn: &TokenTree<S>) -> String {
match tkn {
TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(),
TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(),