Rollup merge of #76285 - matklad:censor-spacing, r=petrochenkov

Move jointness censoring to proc_macro

Proc-macro API currently exposes jointness in `Punct` tokens. That is,
`+` in `+one` is **non** joint.

Our lexer produces jointness info for all tokens, so we need to censor
it *somewhere*

Previously we did this in a lexer, but it makes more sense to do this
in a proc-macro server.

r? @petrochenkov

compiler/rustc_ast/src/tokenstream.rs

@@ -403,8 +403,8 @@ pub fn append(&mut self, new_stream: TokenStream) {
         self.index = index;
     }
 
-    pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
-        self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
+    pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
+        self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
     }
 }
 

compiler/rustc_expand/src/proc_macro_server.rs

@@ -47,15 +47,26 @@ fn to_internal(self) -> token::DelimToken {
     }
 }
 
-impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
-    for TokenTree<Group, Punct, Ident, Literal>
+impl
+    FromInternal<(
+        TreeAndJoint,
+        Option<&'_ tokenstream::TokenTree>,
+        &'_ ParseSess,
+        &'_ mut Vec<Self>,
+    )> for TokenTree<Group, Punct, Ident, Literal>
 {
     fn from_internal(
-        ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
+        ((tree, is_joint), look_ahead, sess, stack): (
+            TreeAndJoint,
+            Option<&tokenstream::TokenTree>,
+            &ParseSess,
+            &mut Vec<Self>,
+        ),
     ) -> Self {
         use rustc_ast::token::*;
 
-        let joint = is_joint == Joint;
+        let joint = is_joint == Joint
+            && matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
         let Token { kind, span } = match tree {
             tokenstream::TokenTree::Delimited(span, delim, tts) => {
                 let delimiter = Delimiter::from_internal(delim);
@@ -445,7 +456,8 @@ fn next(
         loop {
             let tree = iter.stack.pop().or_else(|| {
                 let next = iter.cursor.next_with_joint()?;
-                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
+                let lookahead = iter.cursor.look_ahead(0);
+                Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
             })?;
             // A hack used to pass AST fragments to attribute and derive macros
             // as a single nonterminal token instead of a token stream.

compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -262,10 +262,7 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
             }
             _ => {
                 let tt = TokenTree::Token(self.token.take());
-                let mut is_joint = self.bump();
-                if !self.token.is_op() {
-                    is_joint = NonJoint;
-                }
+                let is_joint = self.bump();
                 Ok((tt, is_joint))
             }
         }

compiler/rustc_parse/src/parser/mod.rs

@@ -822,15 +822,15 @@ pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R
         }
 
         let frame = &self.token_cursor.frame;
-        looker(&match frame.tree_cursor.look_ahead(dist - 1) {
+        match frame.tree_cursor.look_ahead(dist - 1) {
             Some(tree) => match tree {
-                TokenTree::Token(token) => token,
+                TokenTree::Token(token) => looker(token),
                 TokenTree::Delimited(dspan, delim, _) => {
-                    Token::new(token::OpenDelim(delim), dspan.open)
+                    looker(&Token::new(token::OpenDelim(delim.clone()), dspan.open))
                 }
             },
-            None => Token::new(token::CloseDelim(frame.delim), frame.span.close),
-        })
+            None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
+        }
     }
 
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.