//! `tt` crate defines a `TokenTree` data structure: this is the interface (both //! input and output) of macros. It closely mirrors `proc_macro` crate's //! `TokenTree`. #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] use std::fmt; use stdx::impl_from; use text_size::{TextRange, TextSize}; pub use smol_str::SmolStr; /// Represents identity of the token. /// /// For hygiene purposes, we need to track which expanded tokens originated from /// which source tokens. We do it by assigning an distinct identity to each /// source token and making sure that identities are preserved during macro /// expansion. #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct TokenId(pub u32); impl fmt::Debug for TokenId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } impl TokenId { pub const UNSPECIFIED: TokenId = TokenId(!0); pub const fn unspecified() -> TokenId { Self::UNSPECIFIED } } #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct SpanData { /// The text range of this span, relative to the anchor. /// We need the anchor for incrementality, as storing absolute ranges will require /// recomputation on every change in a file at all times. pub range: TextRange, pub anchor: Anchor, /// The syntax context of the span. pub ctx: Ctx, } impl Span for SpanData { const DUMMY: Self = SpanData { range: TextRange::empty(TextSize::new(0)), anchor: Anchor::DUMMY, ctx: Ctx::DUMMY, }; } pub trait SpanAnchor: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } pub trait Span: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } pub trait SyntaxContext: std::fmt::Debug + Copy + Sized + Eq { const DUMMY: Self; } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum TokenTree { Leaf(Leaf), Subtree(Subtree), } impl_from!(Leaf, Subtree for TokenTree); impl TokenTree { pub const fn empty() -> Self { Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Leaf { Literal(Literal), Punct(Punct), Ident(Ident), } impl Leaf { pub fn span(&self) -> &S { match self { Leaf::Literal(it) => &it.span, Leaf::Punct(it) => &it.span, Leaf::Ident(it) => &it.span, } } } impl_from!(Literal, Punct, Ident for Leaf); #[derive(Clone, PartialEq, Eq, Hash)] pub struct Subtree { pub delimiter: Delimiter, pub token_trees: Vec>, } impl Subtree { pub const fn empty() -> Self { Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } } pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) { self.delimiter.open = f(self.delimiter.open); self.delimiter.close = f(self.delimiter.close); self.token_trees.iter_mut().for_each(|tt| match tt { crate::TokenTree::Leaf(leaf) => match leaf { crate::Leaf::Literal(it) => it.span = f(it.span), crate::Leaf::Punct(it) => it.span = f(it.span), crate::Leaf::Ident(it) => it.span = f(it.span), }, crate::TokenTree::Subtree(s) => s.visit_ids(f), }) } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Delimiter { pub open: S, pub close: S, pub kind: DelimiterKind, } impl Delimiter { pub const UNSPECIFIED: Self = Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible }; pub const fn unspecified() -> Self { Self::UNSPECIFIED } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum DelimiterKind { Parenthesis, Brace, Bracket, Invisible, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Literal { pub text: SmolStr, pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Punct { pub char: char, pub spacing: Spacing, pub span: S, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Spacing { Alone, Joint, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. pub struct Ident { pub text: SmolStr, pub span: S, } impl Ident { pub fn new(text: impl Into, span: S) -> Self { Ident { text: text.into(), span } } } fn print_debug_subtree( f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); let Delimiter { kind, open, close } = &subtree.delimiter; let aux = match kind { DelimiterKind::Invisible => format!("$$ {:?} {:?}", open, close), DelimiterKind::Parenthesis => format!("() {:?} {:?}", open, close), DelimiterKind::Brace => format!("{{}} {:?} {:?}", open, close), DelimiterKind::Bracket => format!("[] {:?} {:?}", open, close), }; if subtree.token_trees.is_empty() { write!(f, "{align}SUBTREE {aux}")?; } else { writeln!(f, "{align}SUBTREE {aux}")?; for (idx, child) in subtree.token_trees.iter().enumerate() { print_debug_token(f, child, level + 1)?; if idx != subtree.token_trees.len() - 1 { writeln!(f)?; } } } Ok(()) } fn print_debug_token( f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize, ) -> fmt::Result { let align = " ".repeat(level); match tkn { TokenTree::Leaf(leaf) => match leaf { Leaf::Literal(lit) => write!(f, "{}LITERAL {} {:?}", align, lit.text, lit.span)?, Leaf::Punct(punct) => write!( f, "{}PUNCH {} [{}] {:?}", align, punct.char, if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, punct.span )?, Leaf::Ident(ident) => write!(f, "{}IDENT {} {:?}", align, ident.text, ident.span)?, }, TokenTree::Subtree(subtree) => { print_debug_subtree(f, subtree, level)?; } } Ok(()) } impl fmt::Debug for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { print_debug_subtree(f, self, 0) } } impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { TokenTree::Leaf(it) => fmt::Display::fmt(it, f), TokenTree::Subtree(it) => fmt::Display::fmt(it, f), } } } impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let (l, r) = match self.delimiter.kind { DelimiterKind::Parenthesis => ("(", ")"), DelimiterKind::Brace => ("{", "}"), DelimiterKind::Bracket => ("[", "]"), DelimiterKind::Invisible => ("", ""), }; f.write_str(l)?; let mut needs_space = false; for tt in &self.token_trees { if needs_space { f.write_str(" ")?; } needs_space = true; match tt { TokenTree::Leaf(Leaf::Punct(p)) => { needs_space = p.spacing == Spacing::Alone; fmt::Display::fmt(p, f)?; } tt => fmt::Display::fmt(tt, f)?, } } f.write_str(r)?; Ok(()) } } impl fmt::Display for Leaf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Leaf::Ident(it) => fmt::Display::fmt(it, f), Leaf::Literal(it) => fmt::Display::fmt(it, f), Leaf::Punct(it) => fmt::Display::fmt(it, f), } } } impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.char, f) } } impl Subtree { /// Count the number of tokens recursively pub fn count(&self) -> usize { let children_count = self .token_trees .iter() .map(|c| match c { TokenTree::Subtree(c) => c.count(), TokenTree::Leaf(_) => 0, }) .sum::(); self.token_trees.len() + children_count } } impl Subtree { /// A simple line string used for debugging pub fn as_debug_string(&self) -> String { let delim = match self.delimiter.kind { DelimiterKind::Brace => ("{", "}"), DelimiterKind::Bracket => ("[", "]"), DelimiterKind::Parenthesis => ("(", ")"), DelimiterKind::Invisible => ("$", "$"), }; let mut res = String::new(); res.push_str(delim.0); let mut last = None; for child in &self.token_trees { let s = match child { TokenTree::Leaf(it) => { let s = match it { Leaf::Literal(it) => it.text.to_string(), Leaf::Punct(it) => it.char.to_string(), Leaf::Ident(it) => it.text.to_string(), }; match (it, last) { (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => { " ".to_string() + &s } (Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => { if punct.spacing == Spacing::Alone { " ".to_string() + &s } else { s } } _ => s, } } TokenTree::Subtree(it) => it.as_debug_string(), }; res.push_str(&s); last = Some(child); } res.push_str(delim.1); res } } pub mod buffer; pub fn pretty(tkns: &[TokenTree]) -> String { fn tokentree_to_text(tkn: &TokenTree) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(), TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Subtree(subtree) => { let content = pretty(&subtree.token_trees); let (open, close) = match subtree.delimiter.kind { DelimiterKind::Brace => ("{", "}"), DelimiterKind::Bracket => ("[", "]"), DelimiterKind::Parenthesis => ("(", ")"), DelimiterKind::Invisible => ("", ""), }; format!("{open}{content}{close}") } } } tkns.iter() .fold((String::new(), true), |(last, last_to_joint), tkn| { let s = [last, tokentree_to_text(tkn)].join(if last_to_joint { "" } else { " " }); let mut is_joint = false; if let TokenTree::Leaf(Leaf::Punct(punct)) = tkn { if punct.spacing == Spacing::Joint { is_joint = true; } } (s, is_joint) }) .0 }