mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Re-enable proc-macros

Browse Source
This commit is contained in:
Lukas Wirth 2023-11-28 16:28:51 +01:00
parent 98cfdde8ba
commit b98597f06d
24 changed files with 787 additions and 493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -1255,6 +1255,9 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
name = "proc-macro-api"
version = "0.0.0"
dependencies = [
"base-db",
"indexmap",
"la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap2",
"object 0.32.0",
"paths",

12
crates/base-db/src/fixture.rs
View File

@ -543,6 +543,9 @@ fn expand(
subtree: &Subtree<SpanData>,
_: Option<&Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<Subtree<SpanData>, ProcMacroExpansionError> {
Ok(subtree.clone())
}
@ -557,6 +560,9 @@ fn expand(
_: &Subtree<SpanData>,
attrs: Option<&Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<Subtree<SpanData>, ProcMacroExpansionError> {
attrs
.cloned()
@ -572,6 +578,9 @@ fn expand(
input: &Subtree<SpanData>,
_: Option<&Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<Subtree<SpanData>, ProcMacroExpansionError> {
fn traverse(input: &Subtree<SpanData>) -> Subtree<SpanData> {
let mut token_trees = vec![];
@ -599,6 +608,9 @@ fn expand(
input: &Subtree<SpanData>,
_: Option<&Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<Subtree<SpanData>, ProcMacroExpansionError> {
return Ok(traverse(input));

3
crates/base-db/src/input.rs
View File

@ -262,6 +262,9 @@ fn expand(
subtree: &tt::Subtree<SpanData>,
attrs: Option<&tt::Subtree<SpanData>>,
env: &Env,
def_site: SpanData,
call_site: SpanData,
mixed_site: SpanData,
) -> Result<tt::Subtree<SpanData>, ProcMacroExpansionError>;
}

2
crates/base-db/src/span.rs
View File

@ -38,6 +38,8 @@ impl SyntaxContextId {
// currently (which kind of makes sense but we need it here!)
pub const SELF_REF: Self =
SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 1) });
/// Used syntax fixups
pub const FAKE: Self = SyntaxContextId(unsafe { InternId::new_unchecked(InternId::MAX - 2) });
pub fn is_root(self) -> bool {
self == Self::ROOT

5
crates/hir-def/src/macro_expansion_tests/mod.rs
View File

@ -16,7 +16,7 @@
use std::{iter, ops::Range, sync};
use base_db::{fixture::WithFixture, ProcMacro, SourceDatabase};
use base_db::{fixture::WithFixture, span::SpanData, ProcMacro, SourceDatabase};
use expect_test::Expect;
use hir_expand::{db::ExpandDatabase, span::SpanMapRef, HirFileIdExt, InFile, MacroFileId};
use stdx::format_to;
@ -307,6 +307,9 @@ fn expand(
subtree: &Subtree,
_: Option<&Subtree>,
_: &base_db::Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<Subtree, base_db::ProcMacroExpansionError> {
let (parse, _) =
::mbe::token_tree_to_syntax_node(subtree, ::mbe::TopEntryPoint::MacroItems);

55
crates/hir-expand/src/db.rs
View File

@ -233,7 +233,17 @@ pub fn expand_speculative(
let speculative_expansion = match loc.def.kind {
MacroDefKind::ProcMacro(expander, ..) => {
tt.delimiter = tt::Delimiter::UNSPECIFIED;
expander.expand(db, loc.def.krate, loc.krate, &tt, attr_arg.as_ref())
let call_site = loc.span(db);
expander.expand(
db,
loc.def.krate,
loc.krate,
&tt,
attr_arg.as_ref(),
call_site,
call_site,
call_site,
)
}
MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => {
pseudo_derive_attr_expansion(&tt, attr_arg.as_ref()?)
@ -398,17 +408,23 @@ fn macro_arg(
MacroCallKind::Attr { ast_id, .. } => ast_id.to_ptr(db).to_node(&root).syntax().clone(),
};
let censor = censor_for_macro_input(&loc, &syntax);
// let mut fixups = fixup::fixup_syntax(&node);
// fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new())));
// let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications(
// &node,
// fixups.token_map,
// fixups.next_id,
// fixups.replace,
// fixups.append,
// );
let mut tt = mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor);
let mut tt = match loc.kind {
MacroCallKind::FnLike { .. } => {
mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor)
}
MacroCallKind::Derive { .. } | MacroCallKind::Attr { .. } => {
// let mut fixups = crate::fixup::fixup_syntax(&syntax);
// fixups.replace.extend(censor.into_iter().map(|node| (node.into(), Vec::new())));
// let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications(
// &node,
// fixups.token_map,
// fixups.next_id,
// fixups.replace,
// fixups.append,
// );
mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor)
}
};
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
@ -658,8 +674,19 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
_ => None,
};
let ExpandResult { value: tt, err } =
expander.expand(db, loc.def.krate, loc.krate, &macro_arg, attr_arg);
let call_site = loc.span(db);
let ExpandResult { value: tt, err } = expander.expand(
db,
loc.def.krate,
loc.krate,
&macro_arg,
attr_arg,
// FIXME
call_site,
call_site,
// FIXME
call_site,
);
// Set a hard limit for the expanded tt
if let Err(value) = check_tt_count(&tt) {

273
crates/hir-expand/src/fixup.rs
View File

@ -2,25 +2,30 @@
//! fix up syntax errors in the code we're passing to them.
use std::mem;
use mbe::{SyntheticToken, SyntheticTokenId, TokenMap};
use base_db::{
span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId},
FileId,
};
use la_arena::RawIdx;
use mbe::TokenMap;
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use syntax::{
ast::{self, AstNode, HasLoopBody},
match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, TextSize,
};
use tt::token_id::Subtree;
use tt::Spacing;
use crate::tt::{Ident, Leaf, Punct, Subtree};
/// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to
/// reverse those changes afterwards, and a token map.
#[derive(Debug, Default)]
pub(crate) struct SyntaxFixups {
pub(crate) append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
pub(crate) replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
pub(crate) append: FxHashMap<SyntaxElement, Vec<Leaf>>,
pub(crate) replace: FxHashMap<SyntaxElement, Vec<()>>,
pub(crate) undo_info: SyntaxFixupUndoInfo,
pub(crate) token_map: TokenMap,
pub(crate) next_id: u32,
}
/// This is the information needed to reverse the fixups.
@ -29,21 +34,25 @@ pub struct SyntaxFixupUndoInfo {
original: Box<[Subtree]>,
}
const EMPTY_ID: SyntheticTokenId = SyntheticTokenId(!0);
// censoring -> just don't convert the node
// replacement -> censor + append
// append -> insert a fake node, here we need to assemble some dummy span that we can figure out how
// to remove later
pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
let mut append = FxHashMap::<SyntaxElement, _>::default();
let mut replace = FxHashMap::<SyntaxElement, _>::default();
let mut preorder = node.preorder();
let mut original = Vec::new();
let mut token_map = TokenMap::default();
let mut next_id = 0;
let dummy_range = TextRange::empty(TextSize::new(0));
let dummy_anchor =
SpanAnchor { file_id: FileId(!0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)) };
let fake_span =
SpanData { range: dummy_range, anchor: dummy_anchor, ctx: SyntaxContextId::FAKE };
while let Some(event) = preorder.next() {
let node = match event {
syntax::WalkEvent::Enter(node) => node,
syntax::WalkEvent::Leave(_) => continue,
};
let syntax::WalkEvent::Enter(node) = event else { continue };
/* TODO
if can_handle_error(&node) && has_error_to_handle(&node) {
// the node contains an error node, we have to completely replace it by something valid
let (original_tree, new_tmap, new_next_id) =
@ -68,6 +77,7 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
preorder.skip_subtree();
continue;
}
*/
// In some other situations, we can fix things by just appending some tokens.
let end_range = TextRange::empty(node.text_range().end());
match_ast! {
@ -76,36 +86,32 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
if it.name_ref().is_none() {
// incomplete field access: some_expr.|
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::IDENT,
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
range: end_range,
id: EMPTY_ID,
},
span: fake_span
}),
]);
}
},
ast::ExprStmt(it) => {
if it.semicolon_token().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::SEMICOLON,
text: ";".into(),
range: end_range,
id: EMPTY_ID,
},
Leaf::Punct(Punct {
char: ';',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
ast::LetStmt(it) => {
if it.semicolon_token().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::SEMICOLON,
text: ";".into(),
range: end_range,
id: EMPTY_ID,
},
Leaf::Punct(Punct {
char: ';',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
@ -117,28 +123,25 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
None => continue,
};
append.insert(if_token.into(), vec![
SyntheticToken {
kind: SyntaxKind::IDENT,
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
range: end_range,
id: EMPTY_ID,
},
span: fake_span
}),
]);
}
if it.then_branch().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::L_CURLY,
text: "{".into(),
range: end_range,
id: EMPTY_ID,
},
SyntheticToken {
kind: SyntaxKind::R_CURLY,
text: "}".into(),
range: end_range,
id: EMPTY_ID,
},
// FIXME: THis should be a subtree no?
Leaf::Punct(Punct {
char: '{',
spacing: Spacing::Alone,
span: fake_span
}),
Leaf::Punct(Punct {
char: '}',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
@ -150,46 +153,42 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
None => continue,
};
append.insert(while_token.into(), vec![
SyntheticToken {
kind: SyntaxKind::IDENT,
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
range: end_range,
id: EMPTY_ID,
},
span: fake_span
}),
]);
}
if it.loop_body().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::L_CURLY,
text: "{".into(),
range: end_range,
id: EMPTY_ID,
},
SyntheticToken {
kind: SyntaxKind::R_CURLY,
text: "}".into(),
range: end_range,
id: EMPTY_ID,
},
// FIXME: THis should be a subtree no?
Leaf::Punct(Punct {
char: '{',
spacing: Spacing::Alone,
span: fake_span
}),
Leaf::Punct(Punct {
char: '}',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
ast::LoopExpr(it) => {
if it.loop_body().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::L_CURLY,
text: "{".into(),
range: end_range,
id: EMPTY_ID,
},
SyntheticToken {
kind: SyntaxKind::R_CURLY,
text: "}".into(),
range: end_range,
id: EMPTY_ID,
},
// FIXME: THis should be a subtree no?
Leaf::Punct(Punct {
char: '{',
spacing: Spacing::Alone,
span: fake_span
}),
Leaf::Punct(Punct {
char: '}',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
@ -201,29 +200,26 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
None => continue
};
append.insert(match_token.into(), vec![
SyntheticToken {
kind: SyntaxKind::IDENT,
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
range: end_range,
id: EMPTY_ID
},
span: fake_span
}),
]);
}
if it.match_arm_list().is_none() {
// No match arms
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::L_CURLY,
text: "{".into(),
range: end_range,
id: EMPTY_ID,
},
SyntheticToken {
kind: SyntaxKind::R_CURLY,
text: "}".into(),
range: end_range,
id: EMPTY_ID,
},
// FIXME: THis should be a subtree no?
Leaf::Punct(Punct {
char: '{',
spacing: Spacing::Alone,
span: fake_span
}),
Leaf::Punct(Punct {
char: '}',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
@ -234,10 +230,15 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
};
let [pat, in_token, iter] = [
(SyntaxKind::UNDERSCORE, "_"),
(SyntaxKind::IN_KW, "in"),
(SyntaxKind::IDENT, "__ra_fixup")
].map(|(kind, text)| SyntheticToken { kind, text: text.into(), range: end_range, id: EMPTY_ID});
"_",
"in",
"__ra_fixup"
].map(|text|
Leaf::Ident(Ident {
text: text.into(),
span: fake_span
}),
);
if it.pat().is_none() && it.in_token().is_none() && it.iterable().is_none() {
append.insert(for_token.into(), vec![pat, in_token, iter]);
@ -248,18 +249,17 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
if it.loop_body().is_none() {
append.insert(node.clone().into(), vec![
SyntheticToken {
kind: SyntaxKind::L_CURLY,
text: "{".into(),
range: end_range,
id: EMPTY_ID,
},
SyntheticToken {
kind: SyntaxKind::R_CURLY,
text: "}".into(),
range: end_range,
id: EMPTY_ID,
},
// FIXME: THis should be a subtree no?
Leaf::Punct(Punct {
char: '{',
spacing: Spacing::Alone,
span: fake_span
}),
Leaf::Punct(Punct {
char: '}',
spacing: Spacing::Alone,
span: fake_span
}),
]);
}
},
@ -270,8 +270,6 @@ pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
SyntaxFixups {
append,
replace,
token_map,
next_id,
undo_info: SyntaxFixupUndoInfo { original: original.into_boxed_slice() },
}
}
@ -288,40 +286,33 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool {
has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c))
}
pub(crate) fn reverse_fixups(
tt: &mut Subtree,
token_map: &TokenMap,
undo_info: &SyntaxFixupUndoInfo,
) {
pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo) {
let tts = std::mem::take(&mut tt.token_trees);
tt.token_trees = tts
.into_iter()
// delete all fake nodes
.filter(|tt| match tt {
tt::TokenTree::Leaf(leaf) => {
token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID)
}
tt::TokenTree::Subtree(st) => {
token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID)
}
})
.flat_map(|tt| match tt {
tt::TokenTree::Subtree(mut tt) => {
reverse_fixups(&mut tt, token_map, undo_info);
SmallVec::from_const([tt.into()])
}
tt::TokenTree::Leaf(leaf) => {
if let Some(id) = token_map.synthetic_token_id(*leaf.span()) {
let original = undo_info.original[id.0 as usize].clone();
if original.delimiter.kind == tt::DelimiterKind::Invisible {
original.token_trees.into()
} else {
SmallVec::from_const([original.into()])
}
} else {
SmallVec::from_const([leaf.into()])
}
}
tt::TokenTree::Leaf(leaf) => leaf.span().ctx != SyntaxContextId::FAKE,
tt::TokenTree::Subtree(st) => st.delimiter.open.ctx != SyntaxContextId::FAKE,
})
// .flat_map(|tt| match tt { TODO
// tt::TokenTree::Subtree(mut tt) => {
// reverse_fixups(&mut tt, undo_info);
// SmallVec::from_const([tt.into()])
// }
// tt::TokenTree::Leaf(leaf) => {
// if let Some(id) = leaf.span().anchor {
// let original = undo_info.original[id.0 as usize].clone();
// if original.delimiter.kind == tt::DelimiterKind::Invisible {
// original.token_trees.into()
// } else {
// SmallVec::from_const([original.into()])
// }
// } else {
// SmallVec::from_const([leaf.into()])
// }
// }
// })
.collect();
}

8
crates/hir-expand/src/proc_macro.rs
View File

@ -1,6 +1,6 @@
//! Proc Macro Expander stub
use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
use base_db::{span::SpanData, CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
use stdx::never;
use crate::{db::ExpandDatabase, tt, ExpandError, ExpandResult};
@ -33,6 +33,9 @@ pub fn expand(
calling_crate: CrateId,
tt: &tt::Subtree,
attr_arg: Option<&tt::Subtree>,
def_site: SpanData,
call_site: SpanData,
mixed_site: SpanData,
) -> ExpandResult<tt::Subtree> {
match self.proc_macro_id {
ProcMacroId(DUMMY_ID) => {
@ -68,7 +71,8 @@ pub fn expand(
let krate_graph = db.crate_graph();
// Proc macros have access to the environment variables of the invoking crate.
let env = &krate_graph[calling_crate].env;
match proc_macro.expander.expand(tt, attr_arg, env) {
match proc_macro.expander.expand(tt, attr_arg, env, def_site, call_site, mixed_site)
{
Ok(t) => ExpandResult::ok(t),
Err(err) => match err {
// Don't discard the item in case something unexpected happened while expanding attributes

24
crates/load-cargo/src/lib.rs
View File

@ -376,16 +376,16 @@ fn expand(
subtree: &tt::Subtree<SpanData>,
attrs: Option<&tt::Subtree<SpanData>>,
env: &Env,
def_site: SpanData,
call_site: SpanData,
mixed_site: SpanData,
) -> Result<tt::Subtree<SpanData>, ProcMacroExpansionError> {
let _ = (subtree, attrs, env);
// let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
// match self.0.expand(subtree, attrs, env) {
// Ok(Ok(subtree)) => Ok(subtree),
// Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
// Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
// }
todo!()
let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
match self.0.expand(subtree, attrs, env, def_site, call_site, mixed_site) {
Ok(Ok(subtree)) => Ok(subtree),
Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
}
}
}
@ -399,6 +399,9 @@ fn expand(
subtree: &tt::Subtree<SpanData>,
_: Option<&tt::Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<tt::Subtree<SpanData>, ProcMacroExpansionError> {
Ok(subtree.clone())
}
@ -414,6 +417,9 @@ fn expand(
_: &tt::Subtree<SpanData>,
_: Option<&tt::Subtree<SpanData>>,
_: &Env,
_: SpanData,
_: SpanData,
_: SpanData,
) -> Result<tt::Subtree<SpanData>, ProcMacroExpansionError> {
Ok(tt::Subtree::empty())
}

2
crates/mbe/src/expander/transcriber.rs
View File

@ -51,7 +51,7 @@ macro_rules! binding_err {
marker(&mut span);
let subtree = tt::Subtree {
delimiter: tt::Delimiter {
// TODO split span
// FIXME split span
open: span,
close: span,
kind: delimiter.kind,

3
crates/mbe/src/lib.rs
View File

@ -34,7 +34,8 @@
pub use crate::{
syntax_bridge::{
map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree,
map_from_syntax_node, parse_exprs_with_sep, parse_to_token_tree,
parse_to_token_tree_static_span, syntax_node_to_token_tree,
syntax_node_to_token_tree_censored, token_tree_to_syntax_node, SpanMapper,
},
token_map::TokenMap,

105
crates/mbe/src/syntax_bridge.rs
View File

@ -63,7 +63,7 @@ fn span_for(
/// Convert the syntax node to a `TokenTree` (what macro
/// will consume).
/// TODO: Flesh out the doc comment more thoroughly
/// FIXME: Flesh out the doc comment more thoroughly
pub fn syntax_node_to_token_tree<Anchor, Ctx, SpanMap>(
node: &SyntaxNode,
map: SpanMap,
@ -179,6 +179,19 @@ pub fn parse_to_token_tree<Anchor, Ctx>(
Some(convert_tokens(&mut conv))
}
/// Convert a string to a `TokenTree`
pub fn parse_to_token_tree_static_span<S>(span: S, text: &str) -> Option<tt::Subtree<S>>
where
S: Span,
{
let lexed = parser::LexedStr::new(text);
if lexed.errors().next().is_some() {
return None;
}
let mut conv = StaticRawConverter { lexed, pos: 0, span };
Some(convert_tokens(&mut conv))
}
/// Split token tree with separate expr: $($e:expr)SEP*
pub fn parse_exprs_with_sep<S: Span>(tt: &tt::Subtree<S>, sep: char) -> Vec<tt::Subtree<S>> {
if tt.token_trees.is_empty() {
@ -213,12 +226,10 @@ pub fn parse_exprs_with_sep<S: Span>(tt: &tt::Subtree<S>, sep: char) -> Vec<tt::
res
}
fn convert_tokens<Anchor, Ctx, C>(conv: &mut C) -> tt::Subtree<SpanData<Anchor, Ctx>>
fn convert_tokens<S, C>(conv: &mut C) -> tt::Subtree<S>
where
C: TokenConverter<Anchor, Ctx>,
Ctx: SyntaxContext,
SpanData<Anchor, Ctx>: Span,
Anchor: Copy,
C: TokenConverter<S>,
S: Span,
{
let entry = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] };
let mut stack = NonEmptyVec::new(entry);
@ -452,6 +463,12 @@ struct RawConverter<'a, Anchor> {
pos: usize,
anchor: Anchor,
}
/// A raw token (straight from lexer) converter that gives every token the same span.
struct StaticRawConverter<'a, S> {
lexed: parser::LexedStr<'a>,
pos: usize,
span: S,
}
trait SrcToken<Ctx>: std::fmt::Debug {
fn kind(&self, ctx: &Ctx) -> SyntaxKind;
@ -461,20 +478,16 @@ trait SrcToken<Ctx>: std::fmt::Debug {
fn to_text(&self, ctx: &Ctx) -> SmolStr;
}
trait TokenConverter<Anchor, Ctx>: Sized {
trait TokenConverter<S>: Sized {
type Token: SrcToken<Self>;
fn convert_doc_comment(
&self,
token: &Self::Token,
span: SpanData<Anchor, Ctx>,
) -> Option<Vec<tt::TokenTree<SpanData<Anchor, Ctx>>>>;
fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option<Vec<tt::TokenTree<S>>>;
fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
fn peek(&self) -> Option<Self::Token>;
fn span_for(&self, range: TextRange) -> SpanData<Anchor, Ctx>;
fn span_for(&self, range: TextRange) -> S;
}
impl<Anchor> SrcToken<RawConverter<'_, Anchor>> for usize {
@ -491,7 +504,22 @@ fn to_text(&self, ctx: &RawConverter<'_, Anchor>) -> SmolStr {
}
}
impl<Anchor: Copy, Ctx: SyntaxContext> TokenConverter<Anchor, Ctx> for RawConverter<'_, Anchor>
impl<S: Span> SrcToken<StaticRawConverter<'_, S>> for usize {
fn kind(&self, ctx: &StaticRawConverter<'_, S>) -> SyntaxKind {
ctx.lexed.kind(*self)
}
fn to_char(&self, ctx: &StaticRawConverter<'_, S>) -> Option<char> {
ctx.lexed.text(*self).chars().next()
}
fn to_text(&self, ctx: &StaticRawConverter<'_, S>) -> SmolStr {
ctx.lexed.text(*self).into()
}
}
impl<Anchor: Copy, Ctx: SyntaxContext> TokenConverter<SpanData<Anchor, Ctx>>
for RawConverter<'_, Anchor>
where
SpanData<Anchor, Ctx>: Span,
{
@ -530,6 +558,41 @@ fn span_for(&self, range: TextRange) -> SpanData<Anchor, Ctx> {
}
}
impl<S> TokenConverter<S> for StaticRawConverter<'_, S>
where
S: Span,
{
type Token = usize;
fn convert_doc_comment(&self, &token: &usize, span: S) -> Option<Vec<tt::TokenTree<S>>> {
let text = self.lexed.text(token);
convert_doc_comment(&doc_comment(text), span)
}
fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
if self.pos == self.lexed.len() {
return None;
}
let token = self.pos;
self.pos += 1;
let range = self.lexed.text_range(token);
let range = TextRange::new(range.start.try_into().ok()?, range.end.try_into().ok()?);
Some((token, range))
}
fn peek(&self) -> Option<Self::Token> {
if self.pos == self.lexed.len() {
return None;
}
Some(self.pos)
}
fn span_for(&self, _: TextRange) -> S {
self.span
}
}
struct Converter<SpanMap> {
current: Option<SyntaxToken>,
preorder: PreorderWithTokens,
@ -596,17 +659,13 @@ fn to_text(&self, _ctx: &Converter<SpanMap>) -> SmolStr {
}
}
impl<Anchor: Copy, Ctx, SpanMap> TokenConverter<Anchor, Ctx> for Converter<SpanMap>
impl<S, SpanMap> TokenConverter<S> for Converter<SpanMap>
where
SpanData<Anchor, Ctx>: Span,
SpanMap: SpanMapper<SpanData<Anchor, Ctx>>,
S: Span,
SpanMap: SpanMapper<S>,
{
type Token = SynToken;
fn convert_doc_comment(
&self,
token: &Self::Token,
span: SpanData<Anchor, Ctx>,
) -> Option<Vec<tt::TokenTree<SpanData<Anchor, Ctx>>>> {
fn convert_doc_comment(&self, token: &Self::Token, span: S) -> Option<Vec<tt::TokenTree<S>>> {
convert_doc_comment(token.token(), span)
}
@ -661,7 +720,7 @@ fn peek(&self) -> Option<Self::Token> {
Some(token)
}
fn span_for(&self, range: TextRange) -> SpanData<Anchor, Ctx> {
fn span_for(&self, range: TextRange) -> S {
self.map.span_for(range)
}
}

7
crates/proc-macro-api/Cargo.toml
View File

@ -25,6 +25,7 @@ tracing.workspace = true
triomphe.workspace = true
memmap2 = "0.5.4"
snap = "1.1.0"
indexmap = "2.1.0"
# local deps
paths.workspace = true
@ -32,5 +33,7 @@ tt.workspace = true
stdx.workspace = true
profile.workspace = true
text-size.workspace = true
# Intentionally *not* depend on anything salsa-related
# base-db.workspace = true
# Ideally this crate would not depend on salsa things, but we need span information here which wraps
# InternIds for the syntax context
base-db.workspace = true
la-arena.workspace = true

39
crates/proc-macro-api/src/lib.rs
View File

@ -11,6 +11,8 @@
mod process;
mod version;
use base_db::span::SpanData;
use indexmap::IndexSet;
use paths::AbsPathBuf;
use std::{fmt, io, sync::Mutex};
use triomphe::Arc;
@ -18,7 +20,7 @@
use serde::{Deserialize, Serialize};
use crate::{
msg::{flat::SerializableSpan, ExpandMacro, FlatTree, PanicMessage},
msg::{ExpandMacro, ExpnGlobals, FlatTree, PanicMessage, HAS_GLOBAL_SPANS},
process::ProcMacroProcessSrv,
};
@ -132,32 +134,49 @@ pub fn kind(&self) -> ProcMacroKind {
self.kind
}
pub fn expand<const L: usize, S: SerializableSpan<L>>(
pub fn expand(
&self,
subtree: &tt::Subtree<S>,
attr: Option<&tt::Subtree<S>>,
subtree: &tt::Subtree<SpanData>,
attr: Option<&tt::Subtree<SpanData>>,
env: Vec<(String, String)>,
) -> Result<Result<tt::Subtree<S>, PanicMessage>, ServerError> {
def_site: SpanData,
call_site: SpanData,
mixed_site: SpanData,
) -> Result<Result<tt::Subtree<SpanData>, PanicMessage>, ServerError> {
let version = self.process.lock().unwrap_or_else(|e| e.into_inner()).version();
let current_dir = env
.iter()
.find(|(name, _)| name == "CARGO_MANIFEST_DIR")
.map(|(_, value)| value.clone());
let mut span_data_table = IndexSet::default();
let def_site = span_data_table.insert_full(def_site).0;
let call_site = span_data_table.insert_full(call_site).0;
let mixed_site = span_data_table.insert_full(mixed_site).0;
let task = ExpandMacro {
macro_body: FlatTree::new(subtree, version),
macro_body: FlatTree::new(subtree, version, &mut span_data_table),
macro_name: self.name.to_string(),
attributes: attr.map(|subtree| FlatTree::new(subtree, version)),
attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)),
lib: self.dylib_path.to_path_buf().into(),
env,
current_dir,
has_global_spans: ExpnGlobals {
serialize: version >= HAS_GLOBAL_SPANS,
def_site,
call_site,
mixed_site,
},
};
let request = msg::Request::ExpandMacro(task);
let response = self.process.lock().unwrap_or_else(|e| e.into_inner()).send_task(request)?;
let response = self
.process
.lock()
.unwrap_or_else(|e| e.into_inner())
.send_task(msg::Request::ExpandMacro(task))?;
match response {
msg::Response::ExpandMacro(it) => {
Ok(it.map(|tree| FlatTree::to_subtree(tree, version)))
Ok(it.map(|tree| FlatTree::to_subtree_resolved(tree, version, &span_data_table)))
}
msg::Response::ListMacros(..) | msg::Response::ApiVersionCheck(..) => {
Err(ServerError { message: "unexpected response".to_string(), io: None })

127
crates/proc-macro-api/src/msg.rs
View File

@ -10,21 +10,15 @@
use crate::ProcMacroKind;
pub use crate::msg::flat::FlatTree;
pub use crate::msg::flat::{FlatTree, TokenId};
// The versions of the server protocol
pub const NO_VERSION_CHECK_VERSION: u32 = 0;
pub const VERSION_CHECK_VERSION: u32 = 1;
pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2;
/// This version changes how spans are encoded, kind of. Prior to this version,
/// spans were represented as a single u32 which effectively forced spans to be
/// token ids. Starting with this version, the span fields are still u32,
/// but if the size of the span is greater than 1 then the span data is encoded in
/// an additional vector where the span represents the offset into that vector.
/// This allows encoding bigger spans while supporting the previous versions.
pub const VARIABLE_SIZED_SPANS: u32 = 2;
pub const HAS_GLOBAL_SPANS: u32 = 3;
pub const CURRENT_API_VERSION: u32 = VARIABLE_SIZED_SPANS;
pub const CURRENT_API_VERSION: u32 = HAS_GLOBAL_SPANS;
#[derive(Debug, Serialize, Deserialize)]
pub enum Request {
@ -66,6 +60,24 @@ pub struct ExpandMacro {
pub env: Vec<(String, String)>,
pub current_dir: Option<String>,
/// marker for serde skip stuff
#[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")]
pub has_global_spans: ExpnGlobals,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ExpnGlobals {
#[serde(skip_serializing)]
pub serialize: bool,
pub def_site: usize,
pub call_site: usize,
pub mixed_site: usize,
}
impl ExpnGlobals {
fn skip_serializing_if(&self) -> bool {
!self.serialize
}
}
pub trait Message: Serialize + DeserializeOwned {
@ -120,38 +132,89 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
Ok(())
}
/*TODO
#[cfg(test)]
mod tests {
use tt::{
Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, SpanAnchor, Subtree,
TokenId, TokenTree,
use base_db::{
span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId},
FileId,
};
use la_arena::RawIdx;
use text_size::{TextRange, TextSize};
use tt::{Delimiter, DelimiterKind, Ident, Leaf, Literal, Punct, Spacing, Subtree, TokenTree};
use super::*;
fn fixture_token_tree() -> Subtree<TokenId> {
let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() };
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into()));
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into()));
fn fixture_token_tree() -> Subtree<SpanData> {
let anchor =
SpanAnchor { file_id: FileId(0), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)) };
let mut subtree = Subtree {
delimiter: Delimiter {
open: SpanData {
range: TextRange::empty(TextSize::new(0)),
anchor,
ctx: SyntaxContextId::ROOT,
},
close: SpanData {
range: TextRange::empty(TextSize::new(13)),
anchor,
ctx: SyntaxContextId::ROOT,
},
kind: DelimiterKind::Invisible,
},
token_trees: Vec::new(),
};
subtree.token_trees.push(TokenTree::Leaf(
Ident {
text: "struct".into(),
span: SpanData {
range: TextRange::at(TextSize::new(0), TextSize::of("struct")),
anchor,
ctx: SyntaxContextId::ROOT,
},
}
.into(),
));
subtree.token_trees.push(TokenTree::Leaf(
Ident {
text: "Foo".into(),
span: SpanData {
range: TextRange::at(TextSize::new(5), TextSize::of("Foo")),
anchor,
ctx: SyntaxContextId::ROOT,
},
}
.into(),
));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal {
text: "Foo".into(),
span: TokenId::DUMMY,
span: SpanData {
range: TextRange::at(TextSize::new(8), TextSize::of("Foo")),
anchor,
ctx: SyntaxContextId::ROOT,
},
})));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct {
char: '@',
span: TokenId::DUMMY,
span: SpanData {
range: TextRange::at(TextSize::new(11), TextSize::of('@')),
anchor,
ctx: SyntaxContextId::ROOT,
},
spacing: Spacing::Joint,
})));
subtree.token_trees.push(TokenTree::Subtree(Subtree {
delimiter: Delimiter {
open: TokenId(2),
close: TokenId::DUMMY,
open: SpanData {
range: TextRange::at(TextSize::new(12), TextSize::of('{')),
anchor,
ctx: SyntaxContextId::ROOT,
},
close: SpanData {
range: TextRange::at(TextSize::new(13), TextSize::of('}')),
anchor,
ctx: SyntaxContextId::ROOT,
},
kind: DelimiterKind::Brace,
},
token_trees: vec![],
@ -162,20 +225,26 @@ fn fixture_token_tree() -> Subtree<TokenId> {
#[test]
fn test_proc_macro_rpc_works() {
let tt = fixture_token_tree();
let mut span_data_table = Default::default();
let task = ExpandMacro {
macro_body: FlatTree::new(&tt, CURRENT_API_VERSION),
macro_body: FlatTree::new(&tt, CURRENT_API_VERSION, &mut span_data_table),
macro_name: Default::default(),
attributes: None,
lib: std::env::current_dir().unwrap(),
env: Default::default(),
current_dir: Default::default(),
has_global_spans: ExpnGlobals {
serialize: true,
def_site: 0,
call_site: 0,
mixed_site: 0,
},
};
let json = serde_json::to_string(&task).unwrap();
// println!("{}", json);
let back: ExpandMacro = serde_json::from_str(&json).unwrap();
assert_eq!(tt, back.macro_body.to_subtree(CURRENT_API_VERSION));
assert_eq!(tt, back.macro_body.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table));
}
}
*/

383
crates/proc-macro-api/src/msg/flat.rs
View File

@ -37,40 +37,40 @@
use std::collections::{HashMap, VecDeque};
use base_db::span::SpanData;
use indexmap::IndexSet;
use serde::{Deserialize, Serialize};
use text_size::TextRange;
use tt::{Span, SyntaxContext};
use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, VARIABLE_SIZED_SPANS};
use crate::msg::ENCODE_CLOSE_SPAN_VERSION;
pub trait SerializableSpan<const L: usize>: Span {
fn into_u32(self) -> [u32; L];
fn from_u32(input: [u32; L]) -> Self;
}
// impl SerializableSpan<1> for tt::TokenId {
// fn into_u32(self) -> [u32; 1] {
// [self.0]
// }
// fn from_u32([input]: [u32; 1]) -> Self {
// tt::TokenId(input)
// }
// }
pub type SpanDataIndexMap = IndexSet<SpanData>;
impl<Anchor, Ctx> SerializableSpan<3> for tt::SpanData<Anchor, Ctx>
where
Anchor: From<u32> + Into<u32>,
Self: Span,
Ctx: SyntaxContext,
{
fn into_u32(self) -> [u32; 3] {
[self.anchor.into(), self.range.start().into(), self.range.end().into()]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct TokenId(pub u32);
impl std::fmt::Debug for TokenId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
fn from_u32([file_id, start, end]: [u32; 3]) -> Self {
tt::SpanData {
anchor: file_id.into(),
range: TextRange::new(start.into(), end.into()),
ctx: Ctx::DUMMY,
}
}
impl TokenId {
pub const DEF_SITE: Self = TokenId(0);
pub const CALL_SITE: Self = TokenId(0);
pub const MIXED_SITE: Self = TokenId(0);
}
impl tt::Span for TokenId {
const DUMMY: Self = TokenId(!0);
type Anchor = ();
fn anchor(self) -> Self::Anchor {
()
}
fn mk(_: Self::Anchor, _: text_size::TextRange) -> Self {
Self::DUMMY
}
}
@ -82,82 +82,41 @@ pub struct FlatTree {
ident: Vec<u32>,
token_tree: Vec<u32>,
text: Vec<String>,
#[serde(skip_serializing_if = "SpanMap::do_serialize")]
#[serde(default)]
span_map: SpanMap,
}
#[derive(Serialize, Deserialize, Debug)]
struct SpanMap {
#[serde(skip_serializing)]
serialize: bool,
span_size: u32,
spans: Vec<u32>,
}
impl Default for SpanMap {
fn default() -> Self {
Self { serialize: false, span_size: 1, spans: Default::default() }
}
}
impl SpanMap {
fn serialize_span<const L: usize, S: SerializableSpan<L>>(&mut self, span: S) -> u32 {
let u32s = span.into_u32();
if L == 1 {
u32s[0]
} else {
let offset = self.spans.len() as u32;
self.spans.extend(u32s);
offset
}
}
fn deserialize_span<const L: usize, S: SerializableSpan<L>>(&self, offset: u32) -> S {
S::from_u32(if L == 1 {
[offset].as_ref().try_into().unwrap()
} else {
self.spans[offset as usize..][..L].try_into().unwrap()
})
}
}
impl SpanMap {
fn do_serialize(&self) -> bool {
self.serialize
}
}
struct SubtreeRepr<const L: usize, S> {
open: S,
close: S,
struct SubtreeRepr {
open: TokenId,
close: TokenId,
kind: tt::DelimiterKind,
tt: [u32; 2],
}
struct LiteralRepr<const L: usize, S> {
id: S,
struct LiteralRepr {
id: TokenId,
text: u32,
}
struct PunctRepr<const L: usize, S> {
id: S,
struct PunctRepr {
id: TokenId,
char: char,
spacing: tt::Spacing,
}
struct IdentRepr<const L: usize, S> {
id: S,
struct IdentRepr {
id: TokenId,
text: u32,
}
impl FlatTree {
pub fn new<const L: usize, S: SerializableSpan<L>>(
subtree: &tt::Subtree<S>,
pub fn new(
subtree: &tt::Subtree<SpanData>,
version: u32,
span_data_table: &mut SpanDataIndexMap,
) -> FlatTree {
let mut w = Writer {
string_table: HashMap::new(),
work: VecDeque::new(),
span_data_table,
subtree: Vec::new(),
literal: Vec::new(),
@ -167,78 +126,111 @@ pub fn new<const L: usize, S: SerializableSpan<L>>(
text: Vec::new(),
};
w.write(subtree);
assert!(L == 1 || version >= VARIABLE_SIZED_SPANS);
let mut span_map = SpanMap {
serialize: version >= VARIABLE_SIZED_SPANS && L != 1,
span_size: L as u32,
spans: Vec::new(),
};
return FlatTree {
FlatTree {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
write_vec(&mut span_map, w.subtree, SubtreeRepr::write_with_close_span)
write_vec(w.subtree, SubtreeRepr::write_with_close_span)
} else {
write_vec(&mut span_map, w.subtree, SubtreeRepr::write)
write_vec(w.subtree, SubtreeRepr::write)
},
literal: write_vec(&mut span_map, w.literal, LiteralRepr::write),
punct: write_vec(&mut span_map, w.punct, PunctRepr::write),
ident: write_vec(&mut span_map, w.ident, IdentRepr::write),
literal: write_vec(w.literal, LiteralRepr::write),
punct: write_vec(w.punct, PunctRepr::write),
ident: write_vec(w.ident, IdentRepr::write),
token_tree: w.token_tree,
text: w.text,
span_map,
};
fn write_vec<T, F: Fn(T, &mut SpanMap) -> [u32; N], const N: usize>(
map: &mut SpanMap,
xs: Vec<T>,
f: F,
) -> Vec<u32> {
xs.into_iter().flat_map(|it| f(it, map)).collect()
}
}
pub fn to_subtree<const L: usize, S: SerializableSpan<L>>(
pub fn new_raw(subtree: &tt::Subtree<TokenId>, version: u32) -> FlatTree {
let mut w = Writer {
string_table: HashMap::new(),
work: VecDeque::new(),
span_data_table: &mut (),
subtree: Vec::new(),
literal: Vec::new(),
punct: Vec::new(),
ident: Vec::new(),
token_tree: Vec::new(),
text: Vec::new(),
};
w.write(subtree);
FlatTree {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
write_vec(w.subtree, SubtreeRepr::write_with_close_span)
} else {
write_vec(w.subtree, SubtreeRepr::write)
},
literal: write_vec(w.literal, LiteralRepr::write),
punct: write_vec(w.punct, PunctRepr::write),
ident: write_vec(w.ident, IdentRepr::write),
token_tree: w.token_tree,
text: w.text,
}
}
pub fn to_subtree_resolved(
self,
version: u32,
) -> tt::Subtree<S> {
assert!((version >= VARIABLE_SIZED_SPANS || L == 1) && L as u32 == self.span_map.span_size);
return Reader {
span_data_table: &SpanDataIndexMap,
) -> tt::Subtree<SpanData> {
Reader {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
read_vec(&self.span_map, self.subtree, SubtreeRepr::read_with_close_span)
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
} else {
read_vec(&self.span_map, self.subtree, SubtreeRepr::read)
read_vec(self.subtree, SubtreeRepr::read)
},
literal: read_vec(&self.span_map, self.literal, LiteralRepr::read),
punct: read_vec(&self.span_map, self.punct, PunctRepr::read),
ident: read_vec(&self.span_map, self.ident, IdentRepr::read),
literal: read_vec(self.literal, LiteralRepr::read),
punct: read_vec(self.punct, PunctRepr::read),
ident: read_vec(self.ident, IdentRepr::read),
token_tree: self.token_tree,
text: self.text,
span_data_table,
}
.read();
.read()
}
fn read_vec<T, F: Fn([u32; N], &SpanMap) -> T, const N: usize>(
map: &SpanMap,
xs: Vec<u32>,
f: F,
) -> Vec<T> {
let mut chunks = xs.chunks_exact(N);
let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap(), map)).collect();
assert!(chunks.remainder().is_empty());
res
pub fn to_subtree_unresolved(self, version: u32) -> tt::Subtree<TokenId> {
Reader {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
} else {
read_vec(self.subtree, SubtreeRepr::read)
},
literal: read_vec(self.literal, LiteralRepr::read),
punct: read_vec(self.punct, PunctRepr::read),
ident: read_vec(self.ident, IdentRepr::read),
token_tree: self.token_tree,
text: self.text,
span_data_table: &(),
}
.read()
}
}
impl<const L: usize, S: SerializableSpan<L>> SubtreeRepr<L, S> {
fn write(self, map: &mut SpanMap) -> [u32; 4] {
fn read_vec<T, F: Fn([u32; N]) -> T, const N: usize>(xs: Vec<u32>, f: F) -> Vec<T> {
let mut chunks = xs.chunks_exact(N);
let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap())).collect();
assert!(chunks.remainder().is_empty());
res
}
fn write_vec<T, F: Fn(T) -> [u32; N], const N: usize>(xs: Vec<T>, f: F) -> Vec<u32> {
xs.into_iter().flat_map(f).collect()
}
impl SubtreeRepr {
fn write(self) -> [u32; 4] {
let kind = match self.kind {
tt::DelimiterKind::Invisible => 0,
tt::DelimiterKind::Parenthesis => 1,
tt::DelimiterKind::Brace => 2,
tt::DelimiterKind::Bracket => 3,
};
[map.serialize_span(self.open), kind, self.tt[0], self.tt[1]]
[self.open.0, kind, self.tt[0], self.tt[1]]
}
fn read([open, kind, lo, len]: [u32; 4], map: &SpanMap) -> Self {
fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
let kind = match kind {
0 => tt::DelimiterKind::Invisible,
1 => tt::DelimiterKind::Parenthesis,
@ -246,24 +238,18 @@ impl<const L: usize, S: SerializableSpan<L>> SubtreeRepr<L, S> {
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr { open: map.deserialize_span(open), close: S::DUMMY, kind, tt: [lo, len] }
SubtreeRepr { open: TokenId(open), close: TokenId(!0), kind, tt: [lo, len] }
}
fn write_with_close_span(self, map: &mut SpanMap) -> [u32; 5] {
fn write_with_close_span(self) -> [u32; 5] {
let kind = match self.kind {
tt::DelimiterKind::Invisible => 0,
tt::DelimiterKind::Parenthesis => 1,
tt::DelimiterKind::Brace => 2,
tt::DelimiterKind::Bracket => 3,
};
[
map.serialize_span(self.open),
map.serialize_span(self.close),
kind,
self.tt[0],
self.tt[1],
]
[self.open.0, self.close.0, kind, self.tt[0], self.tt[1]]
}
fn read_with_close_span([open, close, kind, lo, len]: [u32; 5], map: &SpanMap) -> Self {
fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr {
let kind = match kind {
0 => tt::DelimiterKind::Invisible,
1 => tt::DelimiterKind::Parenthesis,
@ -271,64 +257,86 @@ impl<const L: usize, S: SerializableSpan<L>> SubtreeRepr<L, S> {
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr {
open: map.deserialize_span(open),
close: map.deserialize_span(close),
kind,
tt: [lo, len],
}
SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] }
}
}
impl<const L: usize, S: SerializableSpan<L>> LiteralRepr<L, S> {
fn write(self, map: &mut SpanMap) -> [u32; 2] {
[map.serialize_span(self.id), self.text]
impl LiteralRepr {
fn write(self) -> [u32; 2] {
[self.id.0, self.text]
}
fn read([id, text]: [u32; 2], map: &SpanMap) -> Self {
LiteralRepr { id: map.deserialize_span(id), text }
fn read([id, text]: [u32; 2]) -> LiteralRepr {
LiteralRepr { id: TokenId(id), text }
}
}
impl<const L: usize, S: SerializableSpan<L>> PunctRepr<L, S> {
fn write(self, map: &mut SpanMap) -> [u32; 3] {
impl PunctRepr {
fn write(self) -> [u32; 3] {
let spacing = match self.spacing {
tt::Spacing::Alone => 0,
tt::Spacing::Joint => 1,
};
[map.serialize_span(self.id), self.char as u32, spacing]
[self.id.0, self.char as u32, spacing]
}
fn read([id, char, spacing]: [u32; 3], map: &SpanMap) -> Self {
fn read([id, char, spacing]: [u32; 3]) -> PunctRepr {
let spacing = match spacing {
0 => tt::Spacing::Alone,
1 => tt::Spacing::Joint,
other => panic!("bad spacing {other}"),
};
PunctRepr { id: map.deserialize_span(id), char: char.try_into().unwrap(), spacing }
PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing }
}
}
impl<const L: usize, S: SerializableSpan<L>> IdentRepr<L, S> {
fn write(self, map: &mut SpanMap) -> [u32; 2] {
[map.serialize_span(self.id), self.text]
impl IdentRepr {
fn write(self) -> [u32; 2] {
[self.id.0, self.text]
}
fn read(data: [u32; 2], map: &SpanMap) -> Self {
IdentRepr { id: map.deserialize_span(data[0]), text: data[1] }
fn read(data: [u32; 2]) -> IdentRepr {
IdentRepr { id: TokenId(data[0]), text: data[1] }
}
}
struct Writer<'a, const L: usize, S> {
trait Span: Copy {
type Table;
fn token_id_of(table: &mut Self::Table, s: Self) -> TokenId;
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self;
}
impl Span for TokenId {
type Table = ();
fn token_id_of((): &mut Self::Table, token_id: Self) -> TokenId {
token_id
}
fn span_for_token_id((): &Self::Table, id: TokenId) -> Self {
id
}
}
impl Span for SpanData {
type Table = IndexSet<SpanData>;
fn token_id_of(table: &mut Self::Table, span: Self) -> TokenId {
TokenId(table.insert_full(span).0 as u32)
}
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self {
*table.get_index(id.0 as usize).unwrap_or_else(|| &table[0])
}
}
struct Writer<'a, 'span, S: Span> {
work: VecDeque<(usize, &'a tt::Subtree<S>)>,
string_table: HashMap<&'a str, u32>,
span_data_table: &'span mut S::Table,
subtree: Vec<SubtreeRepr<L, S>>,
literal: Vec<LiteralRepr<L, S>>,
punct: Vec<PunctRepr<L, S>>,
ident: Vec<IdentRepr<L, S>>,
subtree: Vec<SubtreeRepr>,
literal: Vec<LiteralRepr>,
punct: Vec<PunctRepr>,
ident: Vec<IdentRepr>,
token_tree: Vec<u32>,
text: Vec<String>,
}
impl<'a, const L: usize, S: Copy> Writer<'a, L, S> {
impl<'a, 'span, S: Span> Writer<'a, 'span, S> {
fn write(&mut self, root: &'a tt::Subtree<S>) {
self.enqueue(root);
while let Some((idx, subtree)) = self.work.pop_front() {
@ -336,6 +344,10 @@ fn write(&mut self, root: &'a tt::Subtree<S>) {
}
}
fn token_id_of(&mut self, span: S) -> TokenId {
S::token_id_of(self.span_data_table, span)
}
fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree<S>) {
let mut first_tt = self.token_tree.len();
let n_tt = subtree.token_trees.len();
@ -353,22 +365,21 @@ fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree<S>) {
tt::Leaf::Literal(lit) => {
let idx = self.literal.len() as u32;
let text = self.intern(&lit.text);
self.literal.push(LiteralRepr { id: lit.span, text });
let id = self.token_id_of(lit.span);
self.literal.push(LiteralRepr { id, text });
idx << 2 | 0b01
}
tt::Leaf::Punct(punct) => {
let idx = self.punct.len() as u32;
self.punct.push(PunctRepr {
char: punct.char,
spacing: punct.spacing,
id: punct.span,
});
let id = self.token_id_of(punct.span);
self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id });
idx << 2 | 0b10
}
tt::Leaf::Ident(ident) => {
let idx = self.ident.len() as u32;
let text = self.intern(&ident.text);
self.ident.push(IdentRepr { id: ident.span, text });
let id = self.token_id_of(ident.span);
self.ident.push(IdentRepr { id, text });
idx << 2 | 0b11
}
},
@ -380,8 +391,8 @@ fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree<S>) {
fn enqueue(&mut self, subtree: &'a tt::Subtree<S>) -> u32 {
let idx = self.subtree.len();
let open = subtree.delimiter.open;
let close = subtree.delimiter.close;
let open = self.token_id_of(subtree.delimiter.open);
let close = self.token_id_of(subtree.delimiter.close);
let delimiter_kind = subtree.delimiter.kind;
self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] });
self.work.push_back((idx, subtree));
@ -398,23 +409,29 @@ pub(crate) fn intern(&mut self, text: &'a str) -> u32 {
}
}
struct Reader<const L: usize, S> {
subtree: Vec<SubtreeRepr<L, S>>,
literal: Vec<LiteralRepr<L, S>>,
punct: Vec<PunctRepr<L, S>>,
ident: Vec<IdentRepr<L, S>>,
struct Reader<'span, S: Span> {
subtree: Vec<SubtreeRepr>,
literal: Vec<LiteralRepr>,
punct: Vec<PunctRepr>,
ident: Vec<IdentRepr>,
token_tree: Vec<u32>,
text: Vec<String>,
span_data_table: &'span S::Table,
}
impl<const L: usize, S: SerializableSpan<L>> Reader<L, S> {
impl<'span, S: Span> Reader<'span, S> {
pub(crate) fn read(self) -> tt::Subtree<S> {
let mut res: Vec<Option<tt::Subtree<S>>> = vec![None; self.subtree.len()];
let read_span = |id| S::span_for_token_id(self.span_data_table, id);
for i in (0..self.subtree.len()).rev() {
let repr = &self.subtree[i];
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
let s = tt::Subtree {
delimiter: tt::Delimiter { open: repr.open, close: repr.close, kind: repr.kind },
delimiter: tt::Delimiter {
open: read_span(repr.open),
close: read_span(repr.close),
kind: repr.kind,
},
token_trees: token_trees
.iter()
.copied()
@ -429,7 +446,7 @@ pub(crate) fn read(self) -> tt::Subtree<S> {
let repr = &self.literal[idx];
tt::Leaf::Literal(tt::Literal {
text: self.text[repr.text as usize].as_str().into(),
span: repr.id,
span: read_span(repr.id),
})
.into()
}
@ -438,7 +455,7 @@ pub(crate) fn read(self) -> tt::Subtree<S> {
tt::Leaf::Punct(tt::Punct {
char: repr.char,
spacing: repr.spacing,
span: repr.id,
span: read_span(repr.id),
})
.into()
}
@ -446,7 +463,7 @@ pub(crate) fn read(self) -> tt::Subtree<S> {
let repr = &self.ident[idx];
tt::Leaf::Ident(tt::Ident {
text: self.text[repr.text as usize].as_str().into(),
span: repr.id,
span: read_span(repr.id),
})
.into()
}

6
crates/proc-macro-srv-cli/src/main.rs
View File

@ -18,14 +18,12 @@ fn main() -> std::io::Result<()> {
run()
}
#[cfg(not(FALSE))]
#[cfg(not(feature = "sysroot-abi"))]
#[cfg(not(any(feature = "sysroot-abi", rust_analyzer)))]
fn run() -> io::Result<()> {
panic!("proc-macro-srv-cli requires the `sysroot-abi` feature to be enabled");
}
#[cfg(FALSE)]
#[cfg(feature = "sysroot-abi")]
#[cfg(any(feature = "sysroot-abi", rust_analyzer))]
fn run() -> io::Result<()> {
use proc_macro_api::msg::{self, Message};

10
crates/proc-macro-srv/src/dylib.rs
View File

@ -11,7 +11,7 @@
use memmap2::Mmap;
use object::Object;
use paths::AbsPath;
use proc_macro_api::{read_dylib_info, ProcMacroKind};
use proc_macro_api::{msg::TokenId, read_dylib_info, ProcMacroKind};
const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";
@ -152,8 +152,14 @@ pub fn expand(
macro_name: &str,
macro_body: &crate::tt::Subtree,
attributes: Option<&crate::tt::Subtree>,
def_site: TokenId,
call_site: TokenId,
mixed_site: TokenId,
) -> Result<crate::tt::Subtree, String> {
let result = self.inner.proc_macros.expand(macro_name, macro_body, attributes);
let result = self
.inner
.proc_macros
.expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site);
result.map_err(|e| e.as_str().unwrap_or_else(|| "<unknown error>".to_string()))
}

38
crates/proc-macro-srv/src/lib.rs
View File

@ -10,7 +10,6 @@
//! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable`
//! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)…
#![cfg(FALSE)] // TODO
#![cfg(any(feature = "sysroot-abi", rust_analyzer))]
#![feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span)]
#![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)]
@ -32,12 +31,25 @@
time::SystemTime,
};
use ::tt::Span;
use proc_macro_api::{
msg::{self, CURRENT_API_VERSION},
msg::{self, ExpnGlobals, TokenId, CURRENT_API_VERSION, HAS_GLOBAL_SPANS},
ProcMacroKind,
};
use ::tt::token_id as tt;
mod tt {
pub use proc_macro_api::msg::TokenId;
pub use ::tt::*;
pub type Subtree = ::tt::Subtree<TokenId>;
pub type TokenTree = ::tt::TokenTree<TokenId>;
pub type Delimiter = ::tt::Delimiter<TokenId>;
pub type Leaf = ::tt::Leaf<TokenId>;
pub type Literal = ::tt::Literal<TokenId>;
pub type Punct = ::tt::Punct<TokenId>;
pub type Ident = ::tt::Ident<TokenId>;
}
// see `build.rs`
include!(concat!(env!("OUT_DIR"), "/rustc_version.rs"));
@ -71,16 +83,28 @@ pub fn expand(&mut self, task: msg::ExpandMacro) -> Result<msg::FlatTree, msg::P
None => None,
};
let macro_body = task.macro_body.to_subtree(CURRENT_API_VERSION);
let attributes = task.attributes.map(|it| it.to_subtree(CURRENT_API_VERSION));
let ExpnGlobals { def_site, call_site, mixed_site, .. } = task.has_global_spans;
let def_site = TokenId(def_site as u32);
let call_site = TokenId(call_site as u32);
let mixed_site = TokenId(mixed_site as u32);
let macro_body = task.macro_body.to_subtree_unresolved(CURRENT_API_VERSION);
let attributes = task.attributes.map(|it| it.to_subtree_unresolved(CURRENT_API_VERSION));
let result = thread::scope(|s| {
let thread = thread::Builder::new()
.stack_size(EXPANDER_STACK_SIZE)
.name(task.macro_name.clone())
.spawn_scoped(s, || {
expander
.expand(&task.macro_name, &macro_body, attributes.as_ref())
.map(|it| msg::FlatTree::new(&it, CURRENT_API_VERSION))
.expand(
&task.macro_name,
&macro_body,
attributes.as_ref(),
def_site,
call_site,
mixed_site,
)
.map(|it| msg::FlatTree::new_raw(&it, CURRENT_API_VERSION))
});
let res = match thread {
Ok(handle) => handle.join(),

39
crates/proc-macro-srv/src/proc_macros.rs
View File

@ -1,7 +1,7 @@
//! Proc macro ABI
use libloading::Library;
use proc_macro_api::{ProcMacroKind, RustCInfo};
use proc_macro_api::{msg::TokenId, ProcMacroKind, RustCInfo};
use crate::{dylib::LoadProcMacroDylibError, server::SYMBOL_INTERNER, tt};
@ -45,6 +45,9 @@ pub(crate) fn expand(
macro_name: &str,
macro_body: &tt::Subtree,
attributes: Option<&tt::Subtree>,
def_site: TokenId,
call_site: TokenId,
mixed_site: TokenId,
) -> Result<tt::Subtree, crate::PanicMessage> {
let parsed_body = crate::server::TokenStream::with_subtree(macro_body.clone());
@ -59,34 +62,56 @@ pub(crate) fn expand(
} if *trait_name == macro_name => {
let res = client.run(
&proc_macro::bridge::server::SameThread,
crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER },
crate::server::RustAnalyzer {
interner: &SYMBOL_INTERNER,
call_site,
def_site,
mixed_site,
},
parsed_body,
true,
);
return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from);
return res
.map(|it| it.into_subtree(call_site))
.map_err(crate::PanicMessage::from);
}
proc_macro::bridge::client::ProcMacro::Bang { name, client }
if *name == macro_name =>
{
let res = client.run(
&proc_macro::bridge::server::SameThread,
crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER },
crate::server::RustAnalyzer {
interner: &SYMBOL_INTERNER,
call_site,
def_site,
mixed_site,
},
parsed_body,
true,
);
return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from);
return res
.map(|it| it.into_subtree(call_site))
.map_err(crate::PanicMessage::from);
}
proc_macro::bridge::client::ProcMacro::Attr { name, client }
if *name == macro_name =>
{
let res = client.run(
&proc_macro::bridge::server::SameThread,
crate::server::RustAnalyzer { interner: &SYMBOL_INTERNER },
crate::server::RustAnalyzer {
interner: &SYMBOL_INTERNER,
call_site,
def_site,
mixed_site,
},
parsed_attributes,
parsed_body,
true,
);
return res.map(|it| it.into_subtree()).map_err(crate::PanicMessage::from);
return res
.map(|it| it.into_subtree(call_site))
.map_err(crate::PanicMessage::from);
}
_ => continue,
}

47
crates/proc-macro-srv/src/server.rs
View File

@ -11,6 +11,7 @@
use proc_macro::bridge::{self, server};
mod token_stream;
use proc_macro_api::msg::TokenId;
pub use token_stream::TokenStream;
use token_stream::TokenStreamBuilder;
@ -43,6 +44,9 @@ pub struct SourceFile {
pub struct RustAnalyzer {
// FIXME: store span information here.
pub(crate) interner: SymbolInternerRef,
pub call_site: TokenId,
pub def_site: TokenId,
pub mixed_site: TokenId,
}
impl server::Types for RustAnalyzer {
@ -69,7 +73,7 @@ fn literal_from_str(
kind: bridge::LitKind::Err,
symbol: Symbol::intern(self.interner, s),
suffix: None,
span: tt::TokenId::unspecified(),
span: self.call_site,
})
}
@ -83,7 +87,7 @@ fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
stream.is_empty()
}
fn from_str(&mut self, src: &str) -> Self::TokenStream {
src.parse().expect("cannot parse string")
Self::TokenStream::from_str(src, self.call_site).expect("cannot parse string")
}
fn to_string(&mut self, stream: &Self::TokenStream) -> String {
stream.to_string()
@ -280,7 +284,7 @@ fn save_span(&mut self, _span: Self::Span) -> usize {
}
fn recover_proc_macro_span(&mut self, _id: usize) -> Self::Span {
// FIXME stub
tt::TokenId::unspecified()
self.call_site
}
/// Recent feature, not yet in the proc_macro
///
@ -317,15 +321,15 @@ fn subspan(
}
fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span {
// FIXME handle span
tt::TokenId::unspecified()
self.call_site
}
fn end(&mut self, _self_: Self::Span) -> Self::Span {
tt::TokenId::unspecified()
self.call_site
}
fn start(&mut self, _self_: Self::Span) -> Self::Span {
tt::TokenId::unspecified()
self.call_site
}
fn line(&mut self, _span: Self::Span) -> usize {
@ -349,9 +353,9 @@ fn normalize_and_validate_ident(&mut self, string: &str) -> Result<Self::Symbol,
impl server::Server for RustAnalyzer {
fn globals(&mut self) -> bridge::ExpnGlobals<Self::Span> {
bridge::ExpnGlobals {
def_site: Span::unspecified(),
call_site: Span::unspecified(),
mixed_site: Span::unspecified(),
def_site: self.def_site,
call_site: self.call_site,
mixed_site: self.mixed_site,
}
}
@ -422,6 +426,8 @@ fn with_symbol_and_suffix<R>(
#[cfg(test)]
mod tests {
use ::tt::Span;
use super::*;
#[test]
@ -430,16 +436,16 @@ fn test_ra_server_to_string() {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
span: tt::TokenId::unspecified(),
span: tt::TokenId::DUMMY,
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
span: tt::TokenId::unspecified(),
span: tt::TokenId::DUMMY,
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
open: tt::TokenId::DUMMY,
close: tt::TokenId::DUMMY,
kind: tt::DelimiterKind::Brace,
},
token_trees: vec![],
@ -452,33 +458,32 @@ fn test_ra_server_to_string() {
#[test]
fn test_ra_server_from_str() {
use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
open: tt::TokenId::DUMMY,
close: tt::TokenId::DUMMY,
kind: tt::DelimiterKind::Parenthesis,
},
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(),
span: tt::TokenId::unspecified(),
span: tt::TokenId::DUMMY,
}))],
});
let t1 = TokenStream::from_str("(a)").unwrap();
let t1 = TokenStream::from_str("(a)", tt::TokenId::DUMMY).unwrap();
assert_eq!(t1.token_trees.len(), 1);
assert_eq!(t1.token_trees[0], subtree_paren_a);
let t2 = TokenStream::from_str("(a);").unwrap();
let t2 = TokenStream::from_str("(a);", tt::TokenId::DUMMY).unwrap();
assert_eq!(t2.token_trees.len(), 2);
assert_eq!(t2.token_trees[0], subtree_paren_a);
let underscore = TokenStream::from_str("_").unwrap();
let underscore = TokenStream::from_str("_", tt::TokenId::DUMMY).unwrap();
assert_eq!(
underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
span: tt::TokenId::unspecified(),
span: tt::TokenId::DUMMY,
}))
);
}

64
crates/proc-macro-srv/src/server/token_stream.rs
View File

@ -1,5 +1,7 @@
//! TokenStream implementation used by sysroot ABI
use proc_macro_api::msg::TokenId;
use crate::tt::{self, TokenTree};
#[derive(Debug, Default, Clone)]
@ -20,8 +22,15 @@ pub(crate) fn with_subtree(subtree: tt::Subtree) -> Self {
}
}
pub(crate) fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
pub(crate) fn into_subtree(self, call_site: TokenId) -> tt::Subtree {
tt::Subtree {
delimiter: tt::Delimiter {
open: call_site,
close: call_site,
kind: tt::DelimiterKind::Invisible,
},
token_trees: self.token_trees,
}
}
pub(super) fn is_empty(&self) -> bool {
@ -84,7 +93,7 @@ pub(super) struct TokenStreamBuilder {
/// pub(super)lic implementation details for the `TokenStream` type, such as iterators.
pub(super) mod token_stream {
use std::str::FromStr;
use proc_macro_api::msg::TokenId;
use super::{tt, TokenStream, TokenTree};
@ -109,14 +118,15 @@ fn into_iter(self) -> Self::IntoIter {
///
/// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to
/// change these errors into `LexError`s later.
impl FromStr for TokenStream {
type Err = LexError;
#[rustfmt::skip]
impl /*FromStr for*/ TokenStream {
// type Err = LexError;
fn from_str(src: &str) -> Result<TokenStream, LexError> {
let (subtree, _token_map) =
mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
pub(crate) fn from_str(src: &str, call_site: TokenId) -> Result<TokenStream, LexError> {
let subtree =
mbe::parse_to_token_tree_static_span(call_site, src).ok_or("Failed to parse from mbe")?;
let subtree = subtree_replace_token_ids_with_unspecified(subtree);
let subtree = subtree_replace_token_ids_with_call_site(subtree,call_site);
Ok(TokenStream::with_subtree(subtree))
}
}
@ -127,43 +137,39 @@ fn to_string(&self) -> String {
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
fn subtree_replace_token_ids_with_call_site(
subtree: tt::Subtree,
call_site: TokenId,
) -> tt::Subtree {
tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
delimiter: tt::Delimiter { open: call_site, close: call_site, ..subtree.delimiter },
token_trees: subtree
.token_trees
.into_iter()
.map(token_tree_replace_token_ids_with_unspecified)
.map(|it| token_tree_replace_token_ids_with_call_site(it, call_site))
.collect(),
}
}
fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
fn token_tree_replace_token_ids_with_call_site(
tt: tt::TokenTree,
call_site: TokenId,
) -> tt::TokenTree {
match tt {
tt::TokenTree::Leaf(leaf) => {
tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
tt::TokenTree::Leaf(leaf_replace_token_ids_with_call_site(leaf, call_site))
}
tt::TokenTree::Subtree(subtree) => {
tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
tt::TokenTree::Subtree(subtree_replace_token_ids_with_call_site(subtree, call_site))
}
}
}
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
fn leaf_replace_token_ids_with_call_site(leaf: tt::Leaf, call_site: TokenId) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
}
tt::Leaf::Literal(lit) => tt::Leaf::Literal(tt::Literal { span: call_site, ..lit }),
tt::Leaf::Punct(punct) => tt::Leaf::Punct(tt::Punct { span: call_site, ..punct }),
tt::Leaf::Ident(ident) => tt::Leaf::Ident(tt::Ident { span: call_site, ..ident }),
}
}
}

23
crates/proc-macro-srv/src/tests/utils.rs
View File

@ -1,18 +1,19 @@
//! utils used in proc-macro tests
use expect_test::Expect;
use std::str::FromStr;
use proc_macro_api::msg::TokenId;
use tt::Span;
use crate::{dylib, proc_macro_test_dylib_path, ProcMacroSrv};
fn parse_string(code: &str) -> Option<crate::server::TokenStream> {
fn parse_string(code: &str, call_site: TokenId) -> Option<crate::server::TokenStream> {
// This is a bit strange. We need to parse a string into a token stream into
// order to create a tt::SubTree from it in fixtures. `into_subtree` is
// implemented by all the ABIs we have so we arbitrarily choose one ABI to
// write a `parse_string` function for and use that. The tests don't really
// care which ABI we're using as the `into_subtree` function isn't part of
// the ABI and shouldn't change between ABI versions.
crate::server::TokenStream::from_str(code).ok()
crate::server::TokenStream::from_str(code, call_site).ok()
}
pub fn assert_expand(macro_name: &str, ra_fixture: &str, expect: Expect) {
@ -24,12 +25,22 @@ pub fn assert_expand_attr(macro_name: &str, ra_fixture: &str, attr_args: &str, e
}
fn assert_expand_impl(macro_name: &str, input: &str, attr: Option<&str>, expect: Expect) {
let call_site = TokenId::DUMMY;
let path = proc_macro_test_dylib_path();
let expander = dylib::Expander::new(&path).unwrap();
let fixture = parse_string(input).unwrap();
let attr = attr.map(|attr| parse_string(attr).unwrap().into_subtree());
let fixture = parse_string(input, call_site).unwrap();
let attr = attr.map(|attr| parse_string(attr, call_site).unwrap().into_subtree(call_site));
let res = expander.expand(macro_name, &fixture.into_subtree(), attr.as_ref()).unwrap();
let res = expander
.expand(
macro_name,
&fixture.into_subtree(call_site),
attr.as_ref(),
TokenId::DUMMY,
TokenId::DUMMY,
TokenId::DUMMY,
)
.unwrap();
expect.assert_eq(&format!("{res:?}"));
}

2
crates/rust-analyzer/src/integrated_benchmarks.rs
View File

@ -56,7 +56,7 @@ fn integrated_highlighting_benchmark() {
analysis.highlight_as_html(file_id, false).unwrap();
}
profile::init_from("*>1");
profile::init_from("*>100");
{
let _it = stdx::timeit("change");