Auto merge of #14998 - Veykril:eager-mapping, r=Veykril

internal: Lazy eager macros

This PR makes eager macros less eager. We now only eagerly expand the input of them, while the actual expansion of the macro itself now happens like other lazy macros.
This change allows unifying a lot of macro handling between the two now, most of the special casing now happens for `include!` specifically as it is a very unique macro (by having two inputs that come from differing files).
Fixes https://github.com/rust-lang/rust-analyzer/issues/14841
Fixes https://github.com/rust-lang/rust-analyzer/issues/14996
This commit is contained in:
bors 2023-06-09 11:15:12 +00:00
commit 2796851961
13 changed files with 355 additions and 276 deletions

View File

@ -113,10 +113,10 @@ fn enter_expand_inner(
call_id: MacroCallId,
error: Option<ExpandError>,
) -> ExpandResult<Option<InFile<Parse<SyntaxNode>>>> {
let file_id = call_id.as_file();
let ExpandResult { value, err } = db.parse_or_expand_with_err(file_id);
let macro_file = call_id.as_macro_file();
let ExpandResult { value, err } = db.parse_macro_expansion(macro_file);
ExpandResult { value: Some(InFile::new(file_id, value)), err: error.or(err) }
ExpandResult { value: Some(InFile::new(macro_file.into(), value.0)), err: error.or(err) }
}
pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) {
@ -179,8 +179,8 @@ fn within_limit<F, T: ast::AstNode>(
} else if self.recursion_limit.check(self.recursion_depth as usize + 1).is_err() {
self.recursion_depth = u32::MAX;
cov_mark::hit!(your_stack_belongs_to_me);
return ExpandResult::only_err(ExpandError::Other(
"reached recursion limit during macro expansion".into(),
return ExpandResult::only_err(ExpandError::other(
"reached recursion limit during macro expansion",
));
}

View File

@ -71,7 +71,7 @@ macro_rules! eprintln {
builtin_derive_macro::BuiltinDeriveExpander,
builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander},
db::ExpandDatabase,
eager::expand_eager_macro,
eager::expand_eager_macro_input,
hygiene::Hygiene,
proc_macro::ProcMacroExpander,
AstId, ExpandError, ExpandResult, ExpandTo, HirFileId, InFile, MacroCallId, MacroCallKind,
@ -865,7 +865,7 @@ fn as_call_id_with_errors(
let path = self.value.path().and_then(|path| path::ModPath::from_src(db, path, &h));
let Some(path) = path else {
return Ok(ExpandResult::only_err(ExpandError::Other("malformed macro invocation".into())));
return Ok(ExpandResult::only_err(ExpandError::other("malformed macro invocation")));
};
macro_call_as_call_id_(
@ -913,7 +913,7 @@ fn macro_call_as_call_id_(
let res = if let MacroDefKind::BuiltInEager(..) = def.kind {
let macro_call = InFile::new(call.ast_id.file_id, call.ast_id.to_node(db));
expand_eager_macro(db, krate, macro_call, def, &resolver)?
expand_eager_macro_input(db, krate, macro_call, def, &resolver)?
} else {
ExpandResult {
value: Some(def.as_lazy_macro(

View File

@ -79,7 +79,7 @@ macro_rules! env {() => {}}
#[rustc_builtin_macro]
macro_rules! env {() => {}}
fn main() { "__RA_UNIMPLEMENTED__"; }
fn main() { "UNRESOLVED_ENV_VAR"; }
"##]],
);
}
@ -442,10 +442,6 @@ macro_rules! surprise {
() => { "s" };
}
macro_rules! stuff {
($string:expr) => { concat!($string) };
}
fn main() { concat!(surprise!()); }
"##,
expect![[r##"
@ -456,10 +452,6 @@ macro_rules! surprise {
() => { "s" };
}
macro_rules! stuff {
($string:expr) => { concat!($string) };
}
fn main() { "s"; }
"##]],
);

View File

@ -194,15 +194,15 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MacroItems);
let macro_items = ast::MacroItems::cast(parsed.syntax_node()).ok_or_else(|| {
debug!("derive node didn't parse");
ExpandError::Other("invalid item definition".into())
ExpandError::other("invalid item definition")
})?;
let item = macro_items.items().next().ok_or_else(|| {
debug!("no module item parsed");
ExpandError::Other("no item found".into())
ExpandError::other("no item found")
})?;
let adt = ast::Adt::cast(item.syntax().clone()).ok_or_else(|| {
debug!("expected adt, found: {:?}", item);
ExpandError::Other("expected struct, enum or union".into())
ExpandError::other("expected struct, enum or union")
})?;
let (name, generic_param_list, shape) = match &adt {
ast::Adt::Struct(it) => (
@ -305,7 +305,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
fn name_to_token(token_map: &TokenMap, name: Option<ast::Name>) -> Result<tt::Ident, ExpandError> {
let name = name.ok_or_else(|| {
debug!("parsed item has no name");
ExpandError::Other("missing name".into())
ExpandError::other("missing name")
})?;
let name_token_id =
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);

View File

@ -14,7 +14,8 @@
};
use crate::{
db::ExpandDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
db::ExpandDatabase, name, quote, tt, EagerCallInfo, ExpandError, ExpandResult, MacroCallId,
MacroCallLoc,
};
macro_rules! register_builtin {
@ -49,7 +50,7 @@ pub fn expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let expander = match *self {
$( EagerExpander::$e_kind => $e_expand, )*
};
@ -67,16 +68,9 @@ fn find_by_name(ident: &name::Name) -> Option<Either<BuiltinFnLikeExpander, Eage
};
}
#[derive(Debug)]
pub struct ExpandedEager {
pub(crate) subtree: tt::Subtree,
/// The included file ID of the include macro.
pub(crate) included_file: Option<(FileId, TokenMap)>,
}
impl ExpandedEager {
fn new(subtree: tt::Subtree) -> Self {
ExpandedEager { subtree, included_file: None }
impl EagerExpander {
pub fn is_include(&self) -> bool {
matches!(self, EagerExpander::Include)
}
}
@ -237,18 +231,16 @@ fn format_args_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
format_args_expand_general(db, id, tt, "")
.map(|x| ExpandedEager { subtree: x, included_file: None })
}
fn format_args_nl_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
format_args_expand_general(db, id, tt, "\\n")
.map(|x| ExpandedEager { subtree: x, included_file: None })
}
fn format_args_expand_general(
@ -509,23 +501,23 @@ fn compile_error_expand(
_db: &dyn ExpandDatabase,
_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let err = match &*tt.token_trees {
[tt::TokenTree::Leaf(tt::Leaf::Literal(it))] => match unquote_str(it) {
Some(unquoted) => ExpandError::Other(unquoted.into()),
None => ExpandError::Other("`compile_error!` argument must be a string".into()),
Some(unquoted) => ExpandError::other(unquoted),
None => ExpandError::other("`compile_error!` argument must be a string"),
},
_ => ExpandError::Other("`compile_error!` argument must be a string".into()),
_ => ExpandError::other("`compile_error!` argument must be a string"),
};
ExpandResult { value: ExpandedEager::new(quote! {}), err: Some(err) }
ExpandResult { value: quote! {}, err: Some(err) }
}
fn concat_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut err = None;
let mut text = String::new();
for (i, mut t) in tt.token_trees.iter().enumerate() {
@ -564,14 +556,14 @@ fn concat_expand(
}
}
}
ExpandResult { value: ExpandedEager::new(quote!(#text)), err }
ExpandResult { value: quote!(#text), err }
}
fn concat_bytes_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut bytes = Vec::new();
let mut err = None;
for (i, t) in tt.token_trees.iter().enumerate() {
@ -604,7 +596,7 @@ fn concat_bytes_expand(
}
}
let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
ExpandResult { value: quote!([#ident]), err }
}
fn concat_bytes_expand_subtree(
@ -637,7 +629,7 @@ fn concat_idents_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut err = None;
let mut ident = String::new();
for (i, t) in tt.token_trees.iter().enumerate() {
@ -652,7 +644,7 @@ fn concat_idents_expand(
}
}
let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
ExpandResult { value: quote!(#ident), err }
}
fn relative_file(
@ -665,10 +657,10 @@ fn relative_file(
let path = AnchoredPath { anchor: call_site, path: path_str };
let res = db
.resolve_path(path)
.ok_or_else(|| ExpandError::Other(format!("failed to load file `{path_str}`").into()))?;
.ok_or_else(|| ExpandError::other(format!("failed to load file `{path_str}`")))?;
// Prevent include itself
if res == call_site && !allow_recursion {
Err(ExpandError::Other(format!("recursive inclusion of `{path_str}`").into()))
Err(ExpandError::other(format!("recursive inclusion of `{path_str}`")))
} else {
Ok(res)
}
@ -687,38 +679,37 @@ fn parse_string(tt: &tt::Subtree) -> Result<String, ExpandError> {
fn include_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
let res = (|| {
let path = parse_string(tt)?;
let file_id = relative_file(db, arg_id, &path, false)?;
let (subtree, map) =
parse_to_token_tree(&db.file_text(file_id)).ok_or(mbe::ExpandError::ConversionError)?;
Ok((subtree, map, file_id))
})();
match res {
Ok((subtree, map, file_id)) => {
ExpandResult::ok(ExpandedEager { subtree, included_file: Some((file_id, map)) })
}
Err(e) => ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
),
_tt: &tt::Subtree,
) -> ExpandResult<tt::Subtree> {
match db.include_expand(arg_id) {
Ok((res, _)) => ExpandResult::ok(res.0.clone()),
Err(e) => ExpandResult::new(tt::Subtree::empty(), e),
}
}
pub(crate) fn include_arg_to_tt(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
) -> Result<(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, TokenMap)>, FileId), ExpandError> {
let loc = db.lookup_intern_macro_call(arg_id);
let Some(EagerCallInfo {arg, arg_id: Some(arg_id), .. }) = loc.eager.as_deref() else {
panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager);
};
let path = parse_string(&arg.0)?;
let file_id = relative_file(db, *arg_id, &path, false)?;
let (subtree, map) =
parse_to_token_tree(&db.file_text(file_id)).ok_or(mbe::ExpandError::ConversionError)?;
Ok((triomphe::Arc::new((subtree, map)), file_id))
}
fn include_bytes_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
if let Err(e) = parse_string(tt) {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
);
return ExpandResult::new(tt::Subtree::empty(), e);
}
// FIXME: actually read the file here if the user asked for macro expansion
@ -729,22 +720,17 @@ fn include_bytes_expand(
span: tt::TokenId::unspecified(),
}))],
};
ExpandResult::ok(ExpandedEager::new(res))
ExpandResult::ok(res)
}
fn include_str_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let path = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
// FIXME: we're not able to read excluded files (which is most of them because
@ -754,14 +740,14 @@ fn include_str_expand(
let file_id = match relative_file(db, arg_id, &path, true) {
Ok(file_id) => file_id,
Err(_) => {
return ExpandResult::ok(ExpandedEager::new(quote!("")));
return ExpandResult::ok(quote!(""));
}
};
let text = db.file_text(file_id);
let text = &*text;
ExpandResult::ok(ExpandedEager::new(quote!(#text)))
ExpandResult::ok(quote!(#text))
}
fn get_env_inner(db: &dyn ExpandDatabase, arg_id: MacroCallId, key: &str) -> Option<String> {
@ -773,15 +759,10 @@ fn env_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
let mut err = None;
@ -789,35 +770,28 @@ fn env_expand(
// The only variable rust-analyzer ever sets is `OUT_DIR`, so only diagnose that to avoid
// unnecessary diagnostics for eg. `CARGO_PKG_NAME`.
if key == "OUT_DIR" {
err = Some(ExpandError::Other(
r#"`OUT_DIR` not set, enable "build scripts" to fix"#.into(),
));
err = Some(ExpandError::other(r#"`OUT_DIR` not set, enable "build scripts" to fix"#));
}
// If the variable is unset, still return a dummy string to help type inference along.
// We cannot use an empty string here, because for
// `include!(concat!(env!("OUT_DIR"), "/foo.rs"))` will become
// `include!("foo.rs"), which might go to infinite loop
"__RA_UNIMPLEMENTED__".to_string()
"UNRESOLVED_ENV_VAR".to_string()
});
let expanded = quote! { #s };
ExpandResult { value: ExpandedEager::new(expanded), err }
ExpandResult { value: expanded, err }
}
fn option_env_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
// FIXME: Use `DOLLAR_CRATE` when that works in eager macros.
let expanded = match get_env_inner(db, arg_id, &key) {
@ -825,5 +799,5 @@ fn option_env_expand(
Some(s) => quote! { ::core::option::Option::Some(#s) },
};
ExpandResult::ok(ExpandedEager::new(expanded))
ExpandResult::ok(expanded)
}

View File

@ -14,9 +14,9 @@
use crate::{
ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion,
builtin_fn_macro::EagerExpander, fixup, hygiene::HygieneFrame, tt, BuiltinAttrExpander,
BuiltinDeriveExpander, BuiltinFnLikeExpander, ExpandError, ExpandResult, ExpandTo, HirFileId,
HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile,
ProcMacroExpander,
BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult,
ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId,
MacroDefKind, MacroFile, ProcMacroExpander,
};
/// Total limit on the number of tokens produced by any macro invocation.
@ -53,9 +53,7 @@ fn expand(
match self {
TokenExpander::DeclarativeMacro { mac, .. } => mac.expand(tt).map_err(Into::into),
TokenExpander::Builtin(it) => it.expand(db, id, tt).map_err(Into::into),
TokenExpander::BuiltinEager(it) => {
it.expand(db, id, tt).map_err(Into::into).map(|res| res.subtree)
}
TokenExpander::BuiltinEager(it) => it.expand(db, id, tt).map_err(Into::into),
TokenExpander::BuiltinAttr(it) => it.expand(db, id, tt),
TokenExpander::BuiltinDerive(it) => it.expand(db, id, tt),
TokenExpander::ProcMacro(_) => {
@ -132,6 +130,14 @@ fn macro_arg(
/// Expand macro call to a token tree.
// This query is LRU cached
fn macro_expand(&self, macro_call: MacroCallId) -> ExpandResult<Arc<tt::Subtree>>;
#[salsa::invoke(crate::builtin_fn_macro::include_arg_to_tt)]
fn include_expand(
&self,
arg_id: MacroCallId,
) -> Result<
(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, mbe::TokenMap)>, base_db::FileId),
ExpandError,
>;
/// Special case of the previous query for procedural macros. We can't LRU
/// proc macros, since they are not deterministic in general, and
/// non-determinism breaks salsa in a very, very, very bad way.
@ -281,31 +287,6 @@ fn parse_macro_expansion(
let _p = profile::span("parse_macro_expansion");
let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id);
if let Some(err) = &err {
if tracing::enabled!(tracing::Level::DEBUG) {
// Note:
// The final goal we would like to make all parse_macro success,
// such that the following log will not call anyway.
let loc = db.lookup_intern_macro_call(macro_file.macro_call_id);
let node = loc.to_node(db);
// collect parent information for warning log
let parents = std::iter::successors(loc.kind.file_id().call_node(db), |it| {
it.file_id.call_node(db)
})
.map(|n| format!("{:#}", n.value))
.collect::<Vec<_>>()
.join("\n");
tracing::debug!(
"fail on macro_parse: (reason: {:?} macro_call: {:#}) parents: {}",
err,
node.value,
parents
);
}
}
let expand_to = macro_expand_to(db, macro_file.macro_call_id);
tracing::debug!("expanded = {}", tt.as_debug_string());
@ -320,9 +301,14 @@ fn macro_arg(
db: &dyn ExpandDatabase,
id: MacroCallId,
) -> Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>> {
let arg = db.macro_arg_text(id)?;
let loc = db.lookup_intern_macro_call(id);
if let Some(EagerCallInfo { arg, arg_id: Some(_), error: _ }) = loc.eager.as_deref() {
return Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default())));
}
let arg = db.macro_arg_text(id)?;
let node = SyntaxNode::new_root(arg);
let censor = censor_for_macro_input(&loc, &node);
let mut fixups = fixup::fixup_syntax(&node);
@ -398,7 +384,17 @@ fn macro_arg_text(db: &dyn ExpandDatabase, id: MacroCallId) -> Option<GreenNode>
return None;
}
}
Some(arg.green().into())
if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() {
Some(
mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::Expr)
.0
.syntax_node()
.green()
.into(),
)
} else {
Some(arg.green().into())
}
}
fn macro_def(
@ -445,23 +441,21 @@ fn macro_def(
fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt::Subtree>> {
let _p = profile::span("macro_expand");
let loc = db.lookup_intern_macro_call(id);
if let Some(eager) = &loc.eager {
return ExpandResult { value: eager.arg_or_expansion.clone(), err: eager.error.clone() };
if let Some(EagerCallInfo { arg, arg_id: None, error }) = loc.eager.as_deref() {
// This is an input expansion for an eager macro. These are already pre-expanded
return ExpandResult { value: Arc::new(arg.0.clone()), err: error.clone() };
}
let expander = match db.macro_def(loc.def) {
Ok(it) => it,
// FIXME: This is weird -- we effectively report macro *definition*
// errors lazily, when we try to expand the macro. Instead, they should
// be reported at the definition site when we construct a def map.
// (Note we do report them also at the definition site in the late diagnostic pass)
// FIXME: We should make sure to enforce a variant that invalid macro
// definitions do not get expanders that could reach this call path!
Err(err) => {
return ExpandResult {
value: Arc::new(tt::Subtree {
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}),
err: Some(ExpandError::Other(format!("invalid macro definition: {err}").into())),
err: Some(ExpandError::other(format!("invalid macro definition: {err}"))),
}
}
};
@ -473,13 +467,21 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
token_trees: Vec::new(),
},
),
err: Some(ExpandError::Other(
// FIXME: We should make sure to enforce a variant that invalid macro
// calls do not reach this call path!
err: Some(ExpandError::other(
"invalid token tree"
.into(),
)),
};
};
let ExpandResult { value: mut tt, err } = expander.expand(db, id, &macro_arg.0);
let (arg_tt, arg_tm, undo_info) = &*macro_arg;
let ExpandResult { value: mut tt, mut err } = expander.expand(db, id, arg_tt);
if let Some(EagerCallInfo { error, .. }) = loc.eager.as_deref() {
// FIXME: We should report both errors!
err = error.clone().or(err);
}
// Set a hard limit for the expanded tt
let count = tt.count();
if TOKEN_LIMIT.check(count).is_err() {
@ -488,18 +490,15 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}),
err: Some(ExpandError::Other(
format!(
"macro invocation exceeds token limit: produced {} tokens, limit is {}",
count,
TOKEN_LIMIT.inner(),
)
.into(),
)),
err: Some(ExpandError::other(format!(
"macro invocation exceeds token limit: produced {} tokens, limit is {}",
count,
TOKEN_LIMIT.inner(),
))),
};
}
fixup::reverse_fixups(&mut tt, &macro_arg.1, &macro_arg.2);
fixup::reverse_fixups(&mut tt, arg_tm, undo_info);
ExpandResult { value: Arc::new(tt), err }
}
@ -520,9 +519,8 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<t
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: Vec::new(),
},
err: Some(ExpandError::Other(
err: Some(ExpandError::other(
"invalid token tree"
.into(),
)),
};
};

View File

@ -31,22 +31,24 @@
MacroCallLoc, MacroDefId, MacroDefKind, UnresolvedMacro,
};
pub fn expand_eager_macro(
pub fn expand_eager_macro_input(
db: &dyn ExpandDatabase,
krate: CrateId,
macro_call: InFile<ast::MacroCall>,
def: MacroDefId,
resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
) -> Result<ExpandResult<Option<MacroCallId>>, UnresolvedMacro> {
let MacroDefKind::BuiltInEager(eager, _) = def.kind else {
panic!("called `expand_eager_macro` on non-eager macro def {def:?}")
assert!(matches!(def.kind, MacroDefKind::BuiltInEager(..)));
let token_tree = macro_call.value.token_tree();
let Some(token_tree) = token_tree else {
return Ok(ExpandResult { value: None, err:
Some(ExpandError::other(
"invalid token tree"
)),
});
};
let hygiene = Hygiene::new(db, macro_call.file_id);
let parsed_args = macro_call
.value
.token_tree()
.map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
.unwrap_or_else(tt::Subtree::empty);
let (parsed_args, arg_token_map) = mbe::syntax_node_to_token_tree(token_tree.syntax());
let ast_map = db.ast_id_map(macro_call.file_id);
let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@ -60,41 +62,40 @@ pub fn expand_eager_macro(
def,
krate,
eager: Some(Box::new(EagerCallInfo {
arg_or_expansion: Arc::new(parsed_args.clone()),
included_file: None,
arg: Arc::new((parsed_args, arg_token_map)),
arg_id: None,
error: None,
})),
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr },
});
let parsed_args = mbe::token_tree_to_syntax_node(&parsed_args, mbe::TopEntryPoint::Expr).0;
let ExpandResult { value, mut err } = eager_macro_recur(
let arg_as_expr = match db.macro_arg_text(arg_id) {
Some(it) => it,
None => {
return Ok(ExpandResult {
value: None,
err: Some(ExpandError::other("invalid token tree")),
})
}
};
let ExpandResult { value: expanded_eager_input, err } = eager_macro_recur(
db,
&hygiene,
InFile::new(arg_id.as_file(), parsed_args.syntax_node()),
&Hygiene::new(db, macro_call.file_id),
InFile::new(arg_id.as_file(), SyntaxNode::new_root(arg_as_expr)),
krate,
resolver,
)?;
let Some(value ) = value else {
let Some(expanded_eager_input) = expanded_eager_input else {
return Ok(ExpandResult { value: None, err })
};
let subtree = {
let mut subtree = mbe::syntax_node_to_token_tree(&value).0;
subtree.delimiter = crate::tt::Delimiter::unspecified();
subtree
};
let res = eager.expand(db, arg_id, &subtree);
if err.is_none() {
err = res.err;
}
let (mut subtree, token_map) = mbe::syntax_node_to_token_tree(&expanded_eager_input);
subtree.delimiter = crate::tt::Delimiter::unspecified();
let loc = MacroCallLoc {
def,
krate,
eager: Some(Box::new(EagerCallInfo {
arg_or_expansion: Arc::new(res.value.subtree),
included_file: res.value.included_file,
arg: Arc::new((subtree, token_map)),
arg_id: Some(arg_id),
error: err.clone(),
})),
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to },
@ -118,8 +119,9 @@ fn lazy_expand(
MacroCallKind::FnLike { ast_id: macro_call.with_value(ast_id), expand_to },
);
let file_id = id.as_file();
db.parse_or_expand_with_err(file_id).map(|parse| InFile::new(file_id, parse))
let macro_file = id.as_macro_file();
db.parse_macro_expansion(macro_file).map(|parse| InFile::new(macro_file.into(), parse.0))
}
fn eager_macro_recur(
@ -142,13 +144,13 @@ fn eager_macro_recur(
let def = match child.path().and_then(|path| ModPath::from_src(db, path, hygiene)) {
Some(path) => macro_resolver(path.clone()).ok_or(UnresolvedMacro { path })?,
None => {
error = Some(ExpandError::Other("malformed macro invocation".into()));
error = Some(ExpandError::other("malformed macro invocation"));
continue;
}
};
let ExpandResult { value, err } = match def.kind {
MacroDefKind::BuiltInEager(..) => {
let id = match expand_eager_macro(
let ExpandResult { value, err } = match expand_eager_macro_input(
db,
krate,
curr.with_value(child.clone()),
@ -158,9 +160,17 @@ fn eager_macro_recur(
Ok(it) => it,
Err(err) => return Err(err),
};
id.map(|call| {
call.map(|call| db.parse_or_expand(call.as_file()).clone_for_update())
})
match value {
Some(call) => {
let ExpandResult { value, err: err2 } =
db.parse_macro_expansion(call.as_macro_file());
ExpandResult {
value: Some(value.0.syntax_node().clone_for_update()),
err: err.or(err2),
}
}
None => ExpandResult { value: None, err },
}
}
MacroDefKind::Declarative(_)
| MacroDefKind::BuiltIn(..)
@ -180,7 +190,7 @@ fn eager_macro_recur(
krate,
macro_resolver,
)?;
let err = if err.is_none() { error } else { err };
let err = err.or(error);
ExpandResult { value, err }
}
};

View File

@ -58,7 +58,13 @@ pub enum ExpandError {
UnresolvedProcMacro(CrateId),
Mbe(mbe::ExpandError),
RecursionOverflowPoisoned,
Other(Box<str>),
Other(Box<Box<str>>),
}
impl ExpandError {
pub fn other(msg: impl Into<Box<str>>) -> Self {
ExpandError::Other(Box::new(msg.into()))
}
}
impl From<mbe::ExpandError> for ExpandError {
@ -97,9 +103,15 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
/// The two variants are encoded in a single u32 which are differentiated by the MSB.
/// If the MSB is 0, the value represents a `FileId`, otherwise the remaining 31 bits represent a
/// `MacroCallId`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct HirFileId(u32);
impl fmt::Debug for HirFileId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.repr().fmt(f)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct MacroFile {
pub macro_call_id: MacroCallId,
@ -115,6 +127,7 @@ pub struct MacroFile {
pub struct MacroCallLoc {
pub def: MacroDefId,
pub(crate) krate: CrateId,
/// Some if `def` is a builtin eager macro.
eager: Option<Box<EagerCallInfo>>,
pub kind: MacroCallKind,
}
@ -140,8 +153,10 @@ pub enum MacroDefKind {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct EagerCallInfo {
/// NOTE: This can be *either* the expansion result, *or* the argument to the eager macro!
arg_or_expansion: Arc<tt::Subtree>,
included_file: Option<(FileId, TokenMap)>,
arg: Arc<(tt::Subtree, TokenMap)>,
/// call id of the eager macro's input file. If this is none, macro call containing this call info
/// is an eager macro's input, otherwise it is its output.
arg_id: Option<MacroCallId>,
error: Option<ExpandError>,
}
@ -206,10 +221,15 @@ pub fn original_file(self, db: &dyn db::ExpandDatabase) -> FileId {
HirFileIdRepr::FileId(id) => break id,
HirFileIdRepr::MacroFile(MacroFile { macro_call_id }) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_call_id);
file_id = match loc.eager.as_deref() {
Some(&EagerCallInfo { included_file: Some((file, _)), .. }) => file.into(),
let is_include_expansion = loc.def.is_include()
&& matches!(
loc.eager.as_deref(),
Some(EagerCallInfo { arg_id: Some(_), .. })
);
file_id = match is_include_expansion.then(|| db.include_expand(macro_call_id)) {
Some(Ok((_, file))) => file.into(),
_ => loc.kind.file_id(),
};
}
}
}
}
@ -325,7 +345,17 @@ pub fn is_include_macro(&self, db: &dyn db::ExpandDatabase) -> bool {
match self.macro_file() {
Some(macro_file) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id);
matches!(loc.eager.as_deref(), Some(EagerCallInfo { included_file: Some(..), .. }))
loc.def.is_include()
}
_ => false,
}
}
pub fn is_eager(&self, db: &dyn db::ExpandDatabase) -> bool {
match self.macro_file() {
Some(macro_file) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id);
matches!(loc.eager.as_deref(), Some(EagerCallInfo { .. }))
}
_ => false,
}
@ -423,6 +453,10 @@ pub fn is_attribute(&self) -> bool {
pub fn is_attribute_derive(&self) -> bool {
matches!(self.kind, MacroDefKind::BuiltInAttr(expander, ..) if expander.is_derive())
}
pub fn is_include(&self) -> bool {
matches!(self.kind, MacroDefKind::BuiltInEager(expander, ..) if expander.is_include())
}
}
impl MacroCallLoc {
@ -569,6 +603,10 @@ impl MacroCallId {
pub fn as_file(self) -> HirFileId {
MacroFile { macro_call_id: self }.into()
}
pub fn as_macro_file(self) -> MacroFile {
MacroFile { macro_call_id: self }
}
}
/// ExpansionInfo mainly describes how to map text range between src and expanded macro
@ -662,7 +700,7 @@ pub fn map_token_down(
let token_id = match token_id_in_attr_input {
Some(token_id) => token_id,
// the token is not inside an attribute's input so do the lookup in the macro_arg as usual
// the token is not inside `an attribute's input so do the lookup in the macro_arg as usual
None => {
let relative_range =
token.value.text_range().checked_sub(self.arg.value.text_range().start())?;
@ -694,14 +732,18 @@ pub fn map_token_up(
let call_id = self.expanded.file_id.macro_file()?.macro_call_id;
let loc = db.lookup_intern_macro_call(call_id);
if let Some((file, map)) = loc.eager.and_then(|e| e.included_file) {
// Special case: map tokens from `include!` expansions to the included file
let range = map.first_range_by_token(token_id, token.value.kind())?;
let source = db.parse(file);
// Special case: map tokens from `include!` expansions to the included file
if loc.def.is_include()
&& matches!(loc.eager.as_deref(), Some(EagerCallInfo { arg_id: Some(_), .. }))
{
if let Ok((tt_and_map, file_id)) = db.include_expand(call_id) {
let range = tt_and_map.1.first_range_by_token(token_id, token.value.kind())?;
let source = db.parse(file_id);
let token = source.syntax_node().covering_element(range).into_token()?;
let token = source.syntax_node().covering_element(range).into_token()?;
return Some((InFile::new(file.into(), token), Origin::Call));
return Some((InFile::new(file_id.into(), token), Origin::Call));
}
}
// Attributes are a bit special for us, they have two inputs, the input tokentree and the annotated item.

View File

@ -46,7 +46,7 @@ pub fn expand(
never!("Non-dummy expander even though there are no proc macros");
return ExpandResult::new(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
ExpandError::other("Internal error"),
);
}
};
@ -60,7 +60,7 @@ pub fn expand(
);
return ExpandResult::new(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
ExpandError::other("Internal error"),
);
}
};
@ -75,14 +75,11 @@ pub fn expand(
ProcMacroExpansionError::System(text)
if proc_macro.kind == ProcMacroKind::Attr =>
{
ExpandResult {
value: tt.clone(),
err: Some(ExpandError::Other(text.into())),
}
ExpandResult { value: tt.clone(), err: Some(ExpandError::other(text)) }
}
ProcMacroExpansionError::System(text)
| ProcMacroExpansionError::Panic(text) => {
ExpandResult::new(tt::Subtree::empty(), ExpandError::Other(text.into()))
ExpandResult::new(tt::Subtree::empty(), ExpandError::other(text))
}
},
}

View File

@ -20,8 +20,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -47,8 +49,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -74,8 +78,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -101,8 +107,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -128,8 +136,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -155,8 +165,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -182,8 +194,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,

View File

@ -18,8 +18,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: TYPE_ALIAS,
@ -43,8 +45,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: CONST,
@ -68,8 +72,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: CONST,
@ -95,8 +101,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: ENUM,
@ -122,8 +130,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_DEF,
@ -147,8 +157,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STATIC,
@ -174,8 +186,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -201,8 +215,12 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
2147483648,
hir_file_id: MacroFile(
MacroFile {
macro_call_id: MacroCallId(
0,
),
},
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -228,8 +246,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -257,8 +277,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -286,8 +308,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -311,8 +335,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: TRAIT,
@ -338,8 +364,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: UNION,
@ -365,8 +393,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MODULE,
@ -392,8 +422,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MODULE,
@ -419,8 +451,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_RULES,
@ -444,8 +478,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -471,8 +507,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_RULES,
@ -496,8 +534,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -521,8 +561,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -561,8 +603,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -599,8 +643,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
1,
hir_file_id: FileId(
FileId(
1,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,

View File

@ -86,6 +86,8 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">assert</span> <span class="brace">{</span><span class="brace">}</span>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">asm</span> <span class="brace">{</span><span class="brace">}</span>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">concat</span> <span class="brace">{</span><span class="brace">}</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">toho</span> <span class="brace">{</span>
<span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="operator">=</span><span class="angle">&gt;</span> <span class="parenthesis">(</span><span class="punctuation">$</span>crate<span class="colon">:</span><span class="colon">:</span>panic<span class="punctuation">!</span><span class="parenthesis">(</span><span class="string_literal">"not yet implemented"</span><span class="parenthesis">)</span><span class="parenthesis">)</span><span class="semicolon">;</span>
@ -172,4 +174,5 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">toho</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">fmt"</span><span class="comma macro">,</span> <span class="numeric_literal macro">0</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro unsafe">asm</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"mov eax, </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="none macro">concat</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="comma macro">,</span> <span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="none macro">format_args</span><span class="operator macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="numeric_literal macro">0</span><span class="parenthesis macro">)</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="brace">}</span></code></pre>

View File

@ -432,6 +432,8 @@ macro_rules! panic {}
macro_rules! assert {}
#[rustc_builtin_macro]
macro_rules! asm {}
#[rustc_builtin_macro]
macro_rules! concat {}
macro_rules! toho {
() => ($crate::panic!("not yet implemented"));
@ -518,6 +520,7 @@ fn main() {
toho!("{}fmt", 0);
asm!("mov eax, {0}");
format_args!(concat!("{}"), "{}");
format_args!("{}", format_args!("{}", 0));
}"#,
expect_file!["./test_data/highlight_strings.html"],
false,