From 78f33c0e960a260d02d0f005350b8378d3003c0d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 10 Oct 2022 14:25:14 +0200 Subject: [PATCH] Expand unmatched mbe fragments to reasonable default token trees Currently we expand unmatched fragments by not replacing them at all, leaving us with `$ident`. This trips up the parser or subsequent macro calls. Instead it makes more sense to replace these with some reasonable default depending on the fragment kind which should make more recursive macro calls work better for completions. --- crates/mbe/src/benchmark.rs | 30 +++++++------- crates/mbe/src/expander.rs | 3 +- crates/mbe/src/expander/matcher.rs | 54 +++++++++++++++--------- crates/mbe/src/expander/transcriber.rs | 57 ++++++++++++++++++++++++-- crates/mbe/src/lib.rs | 6 +-- crates/mbe/src/parser.rs | 41 ++++++++++++++++-- 6 files changed, 146 insertions(+), 45 deletions(-) diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index ac691578d88..9c92bae6a19 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -8,7 +8,7 @@ use syntax::{ use test_utils::{bench, bench_fixture, skip_slow_tests}; use crate::{ - parser::{Op, RepeatKind, Separator}, + parser::{MetaVarKind, Op, RepeatKind, Separator}, syntax_node_to_token_tree, DeclarativeMacro, }; @@ -111,35 +111,35 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) { return match op { - Op::Var { kind, .. } => match kind.as_ref().map(|it| it.as_str()) { - Some("ident") => parent.token_trees.push(make_ident("foo")), - Some("ty") => parent.token_trees.push(make_ident("Foo")), - Some("tt") => parent.token_trees.push(make_ident("foo")), - Some("vis") => parent.token_trees.push(make_ident("pub")), - Some("pat") => parent.token_trees.push(make_ident("foo")), - Some("path") => parent.token_trees.push(make_ident("foo")), - Some("literal") => parent.token_trees.push(make_literal("1")), - Some("expr") => parent.token_trees.push(make_ident("foo")), - Some("lifetime") => { + Op::Var { kind, .. } => match kind.as_ref() { + Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), + Some(MetaVarKind::Ty) => parent.token_trees.push(make_ident("Foo")), + Some(MetaVarKind::Tt) => parent.token_trees.push(make_ident("foo")), + Some(MetaVarKind::Vis) => parent.token_trees.push(make_ident("pub")), + Some(MetaVarKind::Pat) => parent.token_trees.push(make_ident("foo")), + Some(MetaVarKind::Path) => parent.token_trees.push(make_ident("foo")), + Some(MetaVarKind::Literal) => parent.token_trees.push(make_literal("1")), + Some(MetaVarKind::Expr) => parent.token_trees.push(make_ident("foo")), + Some(MetaVarKind::Lifetime) => { parent.token_trees.push(make_punct('\'')); parent.token_trees.push(make_ident("a")); } - Some("block") => { + Some(MetaVarKind::Block) => { parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None)) } - Some("item") => { + Some(MetaVarKind::Item) => { parent.token_trees.push(make_ident("fn")); parent.token_trees.push(make_ident("foo")); parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None)); parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None)); } - Some("meta") => { + Some(MetaVarKind::Meta) => { parent.token_trees.push(make_ident("foo")); parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None)); } None => (), - Some(kind) => panic!("Unhandled kind {}", kind), + Some(kind) => panic!("Unhandled kind {:?}", kind), }, Op::Leaf(leaf) => parent.token_trees.push(leaf.clone().into()), Op::Repeat { tokens, kind, separator } => { diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 1e1bfa55055..100ec6bfb93 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -8,7 +8,7 @@ mod transcriber; use rustc_hash::FxHashMap; use syntax::SmolStr; -use crate::{ExpandError, ExpandResult}; +use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; pub(crate) fn expand_rules( rules: &[crate::Rule], @@ -104,6 +104,7 @@ enum Binding { Fragment(Fragment), Nested(Vec), Empty, + Missing(MetaVarKind), } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 139a8cb8cbe..3f656df25f7 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -66,7 +66,7 @@ use syntax::SmolStr; use crate::{ expander::{Binding, Bindings, ExpandResult, Fragment}, - parser::{Op, RepeatKind, Separator}, + parser::{MetaVarKind, Op, RepeatKind, Separator}, tt_iter::TtIter, ExpandError, MetaTemplate, }; @@ -119,6 +119,7 @@ pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree) -> Match { .map(|it| match it { Binding::Fragment(_) => 1, Binding::Empty => 1, + Binding::Missing(_) => 1, Binding::Nested(it) => count(it.iter()), }) .sum() @@ -130,6 +131,7 @@ enum BindingKind { Empty(SmolStr), Optional(SmolStr), Fragment(SmolStr, Fragment), + Missing(SmolStr, MetaVarKind), Nested(usize, usize), } @@ -190,6 +192,10 @@ impl BindingsBuilder { .push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment)))); } + fn push_missing(&mut self, idx: &mut BindingsIdx, var: &SmolStr, kind: MetaVarKind) { + self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Missing(var.clone(), kind)))); + } + fn push_nested(&mut self, parent: &mut BindingsIdx, child: &BindingsIdx) { let BindingsIdx(idx, nidx) = self.copy(child); self.nodes[parent.0].push(LinkNode::Node(Rc::new(BindingKind::Nested(idx, nidx)))); @@ -222,6 +228,9 @@ impl BindingsBuilder { BindingKind::Fragment(name, fragment) => { bindings.inner.insert(name.clone(), Binding::Fragment(fragment.clone())); } + BindingKind::Missing(name, kind) => { + bindings.inner.insert(name.clone(), Binding::Missing(*kind)); + } BindingKind::Nested(idx, nested_idx) => { let mut nested_nodes = Vec::new(); self.collect_nested(*idx, *nested_idx, &mut nested_nodes); @@ -458,9 +467,9 @@ fn match_loop_inner<'t>( } } OpDelimited::Op(Op::Var { kind, name, .. }) => { - if let Some(kind) = kind { + if let &Some(kind) = kind { let mut fork = src.clone(); - let match_res = match_meta_var(kind.as_str(), &mut fork); + let match_res = match_meta_var(kind, &mut fork); match match_res.err { None => { // Some meta variables are optional (e.g. vis) @@ -475,8 +484,15 @@ fn match_loop_inner<'t>( } Some(err) => { res.add_err(err); - if let Some(fragment) = match_res.value { - bindings_builder.push_fragment(&mut item.bindings, name, fragment); + match match_res.value { + Some(fragment) => bindings_builder.push_fragment( + &mut item.bindings, + name, + fragment, + ), + None => { + bindings_builder.push_missing(&mut item.bindings, name, kind) + } } item.is_error = true; error_items.push(item); @@ -668,20 +684,20 @@ fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter<'_>) -> Result<(), ExpandError> { } } -fn match_meta_var(kind: &str, input: &mut TtIter<'_>) -> ExpandResult> { +fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult> { let fragment = match kind { - "path" => parser::PrefixEntryPoint::Path, - "ty" => parser::PrefixEntryPoint::Ty, + MetaVarKind::Path => parser::PrefixEntryPoint::Path, + MetaVarKind::Ty => parser::PrefixEntryPoint::Ty, // FIXME: These two should actually behave differently depending on the edition. // // https://doc.rust-lang.org/edition-guide/rust-2021/or-patterns-macro-rules.html - "pat" | "pat_param" => parser::PrefixEntryPoint::Pat, - "stmt" => parser::PrefixEntryPoint::Stmt, - "block" => parser::PrefixEntryPoint::Block, - "meta" => parser::PrefixEntryPoint::MetaItem, - "item" => parser::PrefixEntryPoint::Item, - "vis" => parser::PrefixEntryPoint::Vis, - "expr" => { + MetaVarKind::Pat | MetaVarKind::PatParam => parser::PrefixEntryPoint::Pat, + MetaVarKind::Stmt => parser::PrefixEntryPoint::Stmt, + MetaVarKind::Block => parser::PrefixEntryPoint::Block, + MetaVarKind::Meta => parser::PrefixEntryPoint::MetaItem, + MetaVarKind::Item => parser::PrefixEntryPoint::Item, + MetaVarKind::Vis => parser::PrefixEntryPoint::Vis, + MetaVarKind::Expr => { // `expr` should not match underscores. // HACK: Macro expansion should not be done using "rollback and try another alternative". // rustc [explicitly checks the next token][0]. @@ -698,17 +714,17 @@ fn match_meta_var(kind: &str, input: &mut TtIter<'_>) -> ExpandResult { let tt_result = match kind { - "ident" => input + MetaVarKind::Ident => input .expect_ident() .map(|ident| tt::Leaf::from(ident.clone()).into()) .map_err(|()| ExpandError::binding_error("expected ident")), - "tt" => input + MetaVarKind::Tt => input .expect_tt() .map_err(|()| ExpandError::binding_error("expected token tree")), - "lifetime" => input + MetaVarKind::Lifetime => input .expect_lifetime() .map_err(|()| ExpandError::binding_error("expected lifetime")), - "literal" => { + MetaVarKind::Literal => { let neg = input.eat_char('-'); input .expect_literal() diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 7bcc84740f1..cbb59ab8e67 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -6,7 +6,7 @@ use tt::{Delimiter, Subtree}; use crate::{ expander::{Binding, Bindings, Fragment}, - parser::{Op, RepeatKind, Separator}, + parser::{MetaVarKind, Op, RepeatKind, Separator}, ExpandError, ExpandResult, MetaTemplate, }; @@ -15,7 +15,7 @@ impl Bindings { self.inner.contains_key(name) } - fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> { + fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result { macro_rules! binding_err { ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) }; } @@ -26,6 +26,7 @@ impl Bindings { nesting_state.hit = true; b = match b { Binding::Fragment(_) => break, + Binding::Missing(_) => break, Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| { nesting_state.at_end = true; binding_err!("could not find nested binding `{name}`") @@ -37,7 +38,55 @@ impl Bindings { }; } match b { - Binding::Fragment(it) => Ok(it), + Binding::Fragment(it) => Ok(it.clone()), + // emit some reasonable default expansion for missing bindings, + // this gives better recovery than emitting the `$fragment-name` verbatim + Binding::Missing(it) => Ok(match it { + MetaVarKind::Stmt => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { + id: tt::TokenId::unspecified(), + char: ';', + spacing: tt::Spacing::Alone, + }))) + } + MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: Some(tt::Delimiter { + id: tt::TokenId::unspecified(), + kind: tt::DelimiterKind::Brace, + }), + token_trees: vec![], + })), + // FIXME: Meta and Item should get proper defaults + MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { + Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: None, + token_trees: vec![], + })) + } + MetaVarKind::Path + | MetaVarKind::Ty + | MetaVarKind::Pat + | MetaVarKind::PatParam + | MetaVarKind::Expr + | MetaVarKind::Ident => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("missing"), + id: tt::TokenId::unspecified(), + }))) + } + MetaVarKind::Lifetime => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("'missing"), + id: tt::TokenId::unspecified(), + }))) + } + MetaVarKind::Literal => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("\"missing\""), + id: tt::TokenId::unspecified(), + }))) + } + }), Binding::Nested(_) => { Err(binding_err!("expected simple binding, found nested binding `{name}`")) } @@ -157,7 +206,7 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe } else { ctx.bindings.get(v, &mut ctx.nesting).map_or_else( |e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, - |b| ExpandResult::ok(b.clone()), + |it| ExpandResult::ok(it), ) } } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 79da84f4a02..c4f0fa20d6d 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -21,7 +21,7 @@ mod token_map; use std::fmt; use crate::{ - parser::{MetaTemplate, Op}, + parser::{MetaTemplate, MetaVarKind, Op}, tt_iter::TtIter, }; @@ -291,9 +291,9 @@ fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> { // Checks that no repetition which could match an empty token // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558 let lsh_is_empty_seq = separator.is_none() && subtree.iter().all(|child_op| { - match child_op { + match *child_op { // vis is optional - Op::Var { kind: Some(kind), .. } => kind == "vis", + Op::Var { kind: Some(kind), .. } => kind == MetaVarKind::Vis, Op::Repeat { kind: parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne, .. diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index acb4be5846d..351c359b73c 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -50,7 +50,7 @@ impl MetaTemplate { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Op { - Var { name: SmolStr, kind: Option, id: tt::TokenId }, + Var { name: SmolStr, kind: Option, id: tt::TokenId }, Ignore { name: SmolStr, id: tt::TokenId }, Index { depth: u32 }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option }, @@ -65,6 +65,24 @@ pub(crate) enum RepeatKind { ZeroOrOne, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum MetaVarKind { + Path, + Ty, + Pat, + PatParam, + Stmt, + Block, + Meta, + Item, + Vis, + Expr, + Ident, + Tt, + Lifetime, + Literal, +} + #[derive(Clone, Debug, Eq)] pub(crate) enum Separator { Literal(tt::Literal), @@ -179,13 +197,30 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul Ok(res) } -fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result, ParseError> { +fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result, ParseError> { if let Mode::Pattern = mode { src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?; let ident = src .expect_ident() .map_err(|()| ParseError::unexpected("missing fragment specifier"))?; - return Ok(Some(ident.text.clone())); + let kind = match ident.text.as_str() { + "path" => MetaVarKind::Path, + "ty" => MetaVarKind::Ty, + "pat" => MetaVarKind::Pat, + "pat_param" => MetaVarKind::PatParam, + "stmt" => MetaVarKind::Stmt, + "block" => MetaVarKind::Block, + "meta" => MetaVarKind::Meta, + "item" => MetaVarKind::Item, + "vis" => MetaVarKind::Vis, + "expr" => MetaVarKind::Expr, + "ident" => MetaVarKind::Ident, + "tt" => MetaVarKind::Tt, + "lifetime" => MetaVarKind::Lifetime, + "literal" => MetaVarKind::Literal, + _ => return Ok(None), + }; + return Ok(Some(kind)); }; Ok(None) }